Skip to content

Commit 03158e8

Browse files
authored
Merge pull request #40 from codefuse-ai/cyzdev
[feat]Add Go extractor source code
2 parents 4e73672 + 4b45807 commit 03158e8

File tree

19 files changed

+7027
-0
lines changed

19 files changed

+7027
-0
lines changed

language/go/extractor/go.mod

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
module alipay.com/code_insight/coref-go-extractor
2+
3+
go 1.18
4+
5+
require (
6+
github.com/deckarep/golang-set v1.8.0
7+
github.com/glebarez/sqlite v1.4.5
8+
github.com/stretchr/testify v1.8.0
9+
golang.org/x/mod v0.7.0
10+
golang.org/x/tools v0.4.0
11+
gorm.io/gorm v1.23.5
12+
)
13+
14+
require (
15+
github.com/davecgh/go-spew v1.1.1 // indirect
16+
github.com/gogo/protobuf v1.3.2 // indirect
17+
github.com/google/gofuzz v1.2.0 // indirect
18+
github.com/json-iterator/go v1.1.12 // indirect
19+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
20+
github.com/modern-go/reflect2 v1.0.2 // indirect
21+
github.com/pmezard/go-difflib v1.0.0 // indirect
22+
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
23+
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
24+
google.golang.org/appengine v1.6.7 // indirect
25+
google.golang.org/protobuf v1.28.1 // indirect
26+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
27+
gopkg.in/inf.v0 v0.9.1 // indirect
28+
gopkg.in/yaml.v3 v3.0.1 // indirect
29+
k8s.io/apimachinery v0.26.2 // indirect
30+
k8s.io/klog/v2 v2.80.1 // indirect
31+
k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
32+
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
33+
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
34+
sigs.k8s.io/yaml v1.3.0 // indirect
35+
)
36+
37+
require (
38+
github.com/glebarez/go-sqlite v1.17.2 // indirect
39+
github.com/go-logr/logr v1.2.3 // indirect
40+
github.com/golang/protobuf v1.5.2 // indirect
41+
github.com/google/uuid v1.3.0 // indirect
42+
github.com/jinzhu/inflection v1.0.0 // indirect
43+
github.com/jinzhu/now v1.1.4 // indirect
44+
github.com/mattn/go-isatty v0.0.16 // indirect
45+
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
46+
golang.org/x/net v0.7.0 // indirect
47+
golang.org/x/sys v0.5.0 // indirect
48+
golang.org/x/term v0.5.0 // indirect
49+
golang.org/x/text v0.7.0 // indirect
50+
gopkg.in/yaml.v2 v2.4.0 // indirect
51+
k8s.io/client-go v0.26.2
52+
modernc.org/libc v1.16.8 // indirect
53+
modernc.org/mathutil v1.4.1
54+
modernc.org/memory v1.1.1 // indirect
55+
modernc.org/sqlite v1.17.2 // indirect
56+
)

language/go/extractor/go.sum

Lines changed: 540 additions & 0 deletions
Large diffs are not rendered by default.

language/go/extractor/main.go

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package main
2+
3+
import (
4+
"log"
5+
6+
"alipay.com/code_insight/coref-go-extractor/src/config"
7+
"alipay.com/code_insight/coref-go-extractor/src/core"
8+
)
9+
10+
// Go-extractor-core is the core program for executing extraction in a given package root.
11+
// The root is assumed to have a go.mod file in its path.
12+
func main() {
13+
config := createConfig()
14+
15+
if err := core.ExtractWithFlags(nil, []string{"./..."}, config); err != nil {
16+
log.Fatalf("Error running go tooling: %s\n", err.Error())
17+
}
18+
}
19+
20+
// createConfig returns a configuration for the go-extractor-core.
21+
func createConfig() *config.Config {
22+
return &config.Config{
23+
Store: createOrmDesc(),
24+
ParseRule: &config.ParseConfig{
25+
NeedTests: true,
26+
NeedScope: false,
27+
NeedModFile: false,
28+
NeedCompile: false,
29+
},
30+
}
31+
}
32+
33+
// createOrmDesc returns a new OrmDesc with configured database settings.
34+
func createOrmDesc() *config.OrmDesc {
35+
return &config.OrmDesc{
36+
DBDialect: "sqlite3",
37+
DBName: "coref_go_src.db",
38+
DBPath: ".",
39+
}
40+
}
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package main
2+
3+
import (
4+
"alipay.com/code_insight/coref-go-extractor/src/config"
5+
"fmt"
6+
"log"
7+
"os"
8+
"time"
9+
10+
"alipay.com/code_insight/coref-go-extractor/src/util"
11+
)
12+
13+
func usage() {
14+
fmt.Fprintf(os.Stderr,
15+
`%s is a wrapper script that installs dependencies and calls the extractor.
16+
In resource-constrained environments, the environment variable SPARROW_EXTRACTOR_GO_MAX_GOROUTINES
17+
(or its legacy alias MAX_GOROUTINES) can be used to limit the number of parallel goroutines
18+
started by the extractor, which reduces CPU and memory requirements. The default value for this
19+
variable is CPU Nums.
20+
The extractor for go tests files<*_test.go | *.test> is extracted by default.
21+
If you don't need extract go tests files, please add extraction flag with ' -ex noneedtests';
22+
however, types' info of target dependencies is not extracted by default, if you want do extraction for
23+
that, use flag with '-ex needdependency', see the demo below.
24+
`, os.Args[0])
25+
fmt.Fprintf(os.Stderr, "Usage:\n\n %s [<flag for extraction>...] [<buildflag For go>...] [--] <fileRoot>\n", os.Args[0])
26+
fmt.Fprintf(os.Stderr, " Example#1: %s -o ./out/coref_go_src.db $src_root \n", os.Args[0])
27+
fmt.Fprintf(os.Stderr, " Example#2: %s -o ./out/coref_go_src.db -ex noneedtests -ex needscope -mod=mod $src_root \n", os.Args[0])
28+
}
29+
30+
type extractionConf struct {
31+
extractor string
32+
extractArgs []string
33+
buildArgs []string
34+
dbPath string
35+
dbName string
36+
srcRoot string
37+
shouldInstallDependency bool
38+
}
39+
40+
// DependencyInstallerMode is an enum describing how dependencies should be installed
41+
type DependencyInstallerMode int
42+
43+
const (
44+
// GoGetNoModules represents dependency installation using `go get` without modules
45+
GoGetNoModules DependencyInstallerMode = iota
46+
// GoGetWithModules represents dependency installation using `go get` with modules
47+
GoGetWithModules
48+
)
49+
50+
type GetMode int
51+
52+
const (
53+
ModUnset GetMode = iota
54+
ModGet
55+
ModMod
56+
ModVendor
57+
)
58+
59+
func main() {
60+
startTime := time.Now()
61+
62+
// os.Args
63+
cfg := parseExtractorRunningParams([]string{"./extractor_cli", "-o", "./out/coref_go_src.db", "-ex", "needmodfile", "-ex", "noneedtests", "."})
64+
log.Printf("ExtractorRunningParams: %+v\n", cfg)
65+
66+
currentDir, err := os.Getwd()
67+
if err != nil {
68+
log.Fatalf("Failed to get current directory: %s", err)
69+
}
70+
71+
// setting up for go env
72+
util.PrepareRunEnv()
73+
74+
// should load project from extraction src root
75+
if err := os.Chdir(cfg.srcRoot); err != nil {
76+
log.Fatalf("Failed to change directory to the extraction source root: %s", err)
77+
}
78+
79+
if cfg.shouldInstallDependency {
80+
handleDependencyInstallation(currentDir)
81+
}
82+
83+
// for compatible with go111
84+
// see detail @https://maelvls.dev/go111module-everywhere/
85+
// disable go mod Load by default.
86+
if err := os.Setenv("GO111MODULE", "off"); err != nil {
87+
log.Fatalf("Failed to set GO111MODULE environment variable: %s", err)
88+
}
89+
90+
log.Printf("Done preparing workspace in %v.\n", time.Since(startTime))
91+
log.Printf("Running extractor '%s' from directory '%s'.\n", cfg.extractor, cfg.srcRoot)
92+
93+
runGoExtraction(cfg)
94+
}
95+
96+
func setExtractorBasicConfig(cfg *extractionConf) *config.Config {
97+
// Initialize the basic configuration with default values
98+
basicConfig := &config.Config{
99+
Store: &config.OrmDesc{
100+
DBDialect: "sqlite3",
101+
DBName: "coref_go_src.db",
102+
DBPath: ".",
103+
},
104+
ParseRule: &config.ParseConfig{
105+
NeedTests: true, // Default to including tests
106+
},
107+
}
108+
109+
// Update the database path and name if provided in the cfg
110+
if cfg.dbPath != "" {
111+
basicConfig.Store.DBPath = cfg.dbPath
112+
}
113+
if cfg.dbName != "" {
114+
basicConfig.Store.DBName = cfg.dbName
115+
}
116+
117+
// Update the parse rules based on the extraction arguments
118+
for _, arg := range cfg.extractArgs {
119+
switch arg {
120+
case "NeedScope":
121+
basicConfig.ParseRule.NeedScope = true
122+
case "NoNeedTests":
123+
basicConfig.ParseRule.NeedTests = false
124+
case "NeedModFile":
125+
basicConfig.ParseRule.NeedModFile = true
126+
case "NeedCompile":
127+
basicConfig.ParseRule.NeedCompile = true
128+
}
129+
}
130+
131+
return basicConfig
132+
}

0 commit comments

Comments
 (0)