[feat]Add Go extractor source code

language/go/extractor/go.mod

@@ -0,0 +1,56 @@
+module alipay.com/code_insight/coref-go-extractor
+
+go 1.18
+
+require (
+	github.com/deckarep/golang-set v1.8.0
+	github.com/glebarez/sqlite v1.4.5
+	github.com/stretchr/testify v1.8.0
+	golang.org/x/mod v0.7.0
+	golang.org/x/tools v0.4.0
+	gorm.io/gorm v1.23.5
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apimachinery v0.26.2 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+require (
+	github.com/glebarez/go-sqlite v1.17.2 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/client-go v0.26.2
+	modernc.org/libc v1.16.8 // indirect
+	modernc.org/mathutil v1.4.1
+	modernc.org/memory v1.1.1 // indirect
+	modernc.org/sqlite v1.17.2 // indirect
+)

language/go/extractor/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"log"
+
+	"alipay.com/code_insight/coref-go-extractor/src/config"
+	"alipay.com/code_insight/coref-go-extractor/src/core"
+)
+
+// Go-extractor-core is the core program for executing extraction in a given package root.
+// The root is assumed to have a go.mod file in its path.
+func main() {
+	config := createConfig()
+
+	if err := core.ExtractWithFlags(nil, []string{"./..."}, config); err != nil {
+		log.Fatalf("Error running go tooling: %s\n", err.Error())
+	}
+}
+
+// createConfig returns a configuration for the go-extractor-core.
+func createConfig() *config.Config {
+	return &config.Config{
+		Store: createOrmDesc(),
+		ParseRule: &config.ParseConfig{
+			NeedTests:   true,
+			NeedScope:   false,
+			NeedModFile: false,
+			NeedCompile: false,
+		},
+	}
+}
+
+// createOrmDesc returns a new OrmDesc with configured database settings.
+func createOrmDesc() *config.OrmDesc {
+	return &config.OrmDesc{
+		DBDialect: "sqlite3",
+		DBName:    "coref_go_src.db",
+		DBPath:    ".",
+	}
+}

language/go/extractor/src/cli/extractor_cli.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"alipay.com/code_insight/coref-go-extractor/src/config"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"alipay.com/code_insight/coref-go-extractor/src/util"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		`%s is a wrapper script that installs dependencies and calls the extractor.
+In resource-constrained environments, the environment variable SPARROW_EXTRACTOR_GO_MAX_GOROUTINES
+(or its legacy alias MAX_GOROUTINES) can be used to limit the number of parallel goroutines
+started by the extractor, which reduces CPU and memory requirements. The default value for this
+variable is CPU Nums.
+The extractor for go tests files<*_test.go | *.test> is extracted by default.
+If you don't need extract go tests files, please add extraction flag with ' -ex noneedtests';
+however, types' info of target dependencies is not extracted by default, if you want do extraction for
+that, use flag with '-ex needdependency', see the demo below.
+`, os.Args[0])
+	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s [<flag for extraction>...] [<buildflag For go>...] [--] <fileRoot>\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "  Example#1: %s -o ./out/coref_go_src.db  $src_root \n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "  Example#2: %s -o ./out/coref_go_src.db -ex noneedtests -ex needscope -mod=mod $src_root \n", os.Args[0])
+}
+
+type extractionConf struct {
+	extractor               string
+	extractArgs             []string
+	buildArgs               []string
+	dbPath                  string
+	dbName                  string
+	srcRoot                 string
+	shouldInstallDependency bool
+}
+
+// DependencyInstallerMode is an enum describing how dependencies should be installed
+type DependencyInstallerMode int
+
+const (
+	// GoGetNoModules represents dependency installation using `go get` without modules
+	GoGetNoModules DependencyInstallerMode = iota
+	// GoGetWithModules represents dependency installation using `go get` with modules
+	GoGetWithModules
+)
+
+type GetMode int
+
+const (
+	ModUnset GetMode = iota
+	ModGet
+	ModMod
+	ModVendor
+)
+
+func main() {
+	startTime := time.Now()
+
+	// os.Args
+	cfg := parseExtractorRunningParams([]string{"./extractor_cli", "-o", "./out/coref_go_src.db", "-ex", "needmodfile", "-ex", "noneedtests", "."})
+	log.Printf("ExtractorRunningParams: %+v\n", cfg)
+
+	currentDir, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Failed to get current directory: %s", err)
+	}
+
+	// setting up for go env
+	util.PrepareRunEnv()
+
+	// should load project from extraction src root
+	if err := os.Chdir(cfg.srcRoot); err != nil {
+		log.Fatalf("Failed to change directory to the extraction source root: %s", err)
+	}
+
+	if cfg.shouldInstallDependency {
+		handleDependencyInstallation(currentDir)
+	}
+
+	// for compatible with go111
+	// see detail @https://maelvls.dev/go111module-everywhere/
+	// disable go mod Load by default.
+	if err := os.Setenv("GO111MODULE", "off"); err != nil {
+		log.Fatalf("Failed to set GO111MODULE environment variable: %s", err)
+	}
+
+	log.Printf("Done preparing workspace in %v.\n", time.Since(startTime))
+	log.Printf("Running extractor '%s' from directory '%s'.\n", cfg.extractor, cfg.srcRoot)
+
+	runGoExtraction(cfg)
+}
+
+func setExtractorBasicConfig(cfg *extractionConf) *config.Config {
+	// Initialize the basic configuration with default values
+	basicConfig := &config.Config{
+		Store: &config.OrmDesc{
+			DBDialect: "sqlite3",
+			DBName:    "coref_go_src.db",
+			DBPath:    ".",
+		},
+		ParseRule: &config.ParseConfig{
+			NeedTests: true, // Default to including tests
+		},
+	}
+
+	// Update the database path and name if provided in the cfg
+	if cfg.dbPath != "" {
+		basicConfig.Store.DBPath = cfg.dbPath
+	}
+	if cfg.dbName != "" {
+		basicConfig.Store.DBName = cfg.dbName
+	}
+
+	// Update the parse rules based on the extraction arguments
+	for _, arg := range cfg.extractArgs {
+		switch arg {
+		case "NeedScope":
+			basicConfig.ParseRule.NeedScope = true
+		case "NoNeedTests":
+			basicConfig.ParseRule.NeedTests = false
+		case "NeedModFile":
+			basicConfig.ParseRule.NeedModFile = true
+		case "NeedCompile":
+			basicConfig.ParseRule.NeedCompile = true
+		}
+	}
+
+	return basicConfig
+}