diff --git a/internal/cache/cache.go b/internal/cache/cache.go index ed52fcf4ac43..c249084e1c53 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "runtime" - "sort" + "slices" + "strings" "sync" + "golang.org/x/exp/maps" "golang.org/x/tools/go/packages" "github.com/golangci/golangci-lint/internal/go/cache" @@ -25,8 +27,13 @@ const ( HashModeNeedAllDeps ) -// Cache is a per-package data cache. A cached data is invalidated when -// package, or it's dependencies change. +var ErrMissing = errors.New("missing data") + +type hashResults map[HashMode]string + +// Cache is a per-package data cache. +// A cached data is invalidated when package, +// or it's dependencies change. type Cache struct { lowLevelCache cache.Cache pkgHashes sync.Map @@ -45,44 +52,24 @@ func NewCache(sw *timeutils.Stopwatch, log logutils.Log) (*Cache, error) { } func (c *Cache) Close() { - c.sw.TrackStage("close", func() { - err := c.lowLevelCache.Close() - if err != nil { - c.log.Errorf("cache close: %v", err) - } - }) + err := c.sw.TrackStageErr("close", c.lowLevelCache.Close) + if err != nil { + c.log.Errorf("cache close: %v", err) + } } func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data any) error { - var err error - buf := &bytes.Buffer{} - c.sw.TrackStage("gob", func() { - err = gob.NewEncoder(buf).Encode(data) - }) + buf, err := c.encode(data) if err != nil { - return fmt.Errorf("failed to gob encode: %w", err) + return err } - var aID cache.ActionID - - c.sw.TrackStage("key build", func() { - aID, err = c.pkgActionID(pkg, mode) - if err == nil { - subkey, subkeyErr := cache.Subkey(aID, key) - if subkeyErr != nil { - err = fmt.Errorf("failed to build subkey: %w", subkeyErr) - } - aID = subkey - } - }) + actionID, err := c.buildKey(pkg, mode, key) if err != nil { return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) } - c.ioSem <- struct{}{} - c.sw.TrackStage("cache io", func() { - err = cache.PutBytes(c.lowLevelCache, aID, buf.Bytes()) - }) - <-c.ioSem + + err = c.putBytes(actionID, buf) if err != nil { return fmt.Errorf("failed to save data to low-level cache by key %s for package %s: %w", key, pkg.Name, err) } @@ -90,31 +77,13 @@ func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data any) return nil } -var ErrMissing = errors.New("missing data") - func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data any) error { - var aID cache.ActionID - var err error - c.sw.TrackStage("key build", func() { - aID, err = c.pkgActionID(pkg, mode) - if err == nil { - subkey, subkeyErr := cache.Subkey(aID, key) - if subkeyErr != nil { - err = fmt.Errorf("failed to build subkey: %w", subkeyErr) - } - aID = subkey - } - }) + actionID, err := c.buildKey(pkg, mode, key) if err != nil { return fmt.Errorf("failed to calculate package %s action id: %w", pkg.Name, err) } - var b []byte - c.ioSem <- struct{}{} - c.sw.TrackStage("cache io", func() { - b, _, err = cache.GetBytes(c.lowLevelCache, aID) - }) - <-c.ioSem + cachedData, err := c.getBytes(actionID) if err != nil { if cache.IsErrMissing(err) { return ErrMissing @@ -122,14 +91,23 @@ func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data any) return fmt.Errorf("failed to get data from low-level cache by key %s for package %s: %w", key, pkg.Name, err) } - c.sw.TrackStage("gob", func() { - err = gob.NewDecoder(bytes.NewReader(b)).Decode(data) - }) - if err != nil { - return fmt.Errorf("failed to gob decode: %w", err) - } + return c.decode(cachedData, data) +} - return nil +func (c *Cache) buildKey(pkg *packages.Package, mode HashMode, key string) (cache.ActionID, error) { + return timeutils.TrackStage(c.sw, "key build", func() (cache.ActionID, error) { + actionID, err := c.pkgActionID(pkg, mode) + if err != nil { + return actionID, err + } + + subkey, subkeyErr := cache.Subkey(actionID, key) + if subkeyErr != nil { + return actionID, fmt.Errorf("failed to build subkey: %w", subkeyErr) + } + + return subkey, nil + }) } func (c *Cache) pkgActionID(pkg *packages.Package, mode HashMode) (cache.ActionID, error) { @@ -142,89 +120,172 @@ func (c *Cache) pkgActionID(pkg *packages.Package, mode HashMode) (cache.ActionI if err != nil { return cache.ActionID{}, fmt.Errorf("failed to make a hash: %w", err) } + fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) fmt.Fprintf(key, "pkghash %s\n", hash) return key.Sum(), nil } -// packageHash computes a package's hash. The hash is based on all Go -// files that make up the package, as well as the hashes of imported -// packages. func (c *Cache) packageHash(pkg *packages.Package, mode HashMode) (string, error) { - type hashResults map[HashMode]string - hashResI, ok := c.pkgHashes.Load(pkg) - if ok { - hashRes := hashResI.(hashResults) - if _, ok := hashRes[mode]; !ok { - return "", fmt.Errorf("no mode %d in hash result", mode) + results, found := c.pkgHashes.Load(pkg) + if found { + hashRes := results.(hashResults) + if result, ok := hashRes[mode]; ok { + return result, nil } - return hashRes[mode], nil + + return "", fmt.Errorf("no mode %d in hash result", mode) } - hashRes := hashResults{} + hashRes, err := c.computePkgHash(pkg) + if err != nil { + return "", err + } + + result, found := hashRes[mode] + if !found { + return "", fmt.Errorf("invalid mode %d", mode) + } + c.pkgHashes.Store(pkg, hashRes) + + return result, nil +} + +// computePkgHash computes a package's hash. +// The hash is based on all Go files that make up the package, +// as well as the hashes of imported packages. +func (c *Cache) computePkgHash(pkg *packages.Package) (hashResults, error) { key, err := cache.NewHash("package hash") if err != nil { - return "", fmt.Errorf("failed to make a hash: %w", err) + return nil, fmt.Errorf("failed to make a hash: %w", err) } + hashRes := hashResults{} + fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath) + for _, f := range pkg.CompiledGoFiles { - c.ioSem <- struct{}{} - h, fErr := cache.FileHash(f) - <-c.ioSem + h, fErr := c.fileHash(f) if fErr != nil { - return "", fmt.Errorf("failed to calculate file %s hash: %w", f, fErr) + return nil, fmt.Errorf("failed to calculate file %s hash: %w", f, fErr) } + fmt.Fprintf(key, "file %s %x\n", f, h) } + curSum := key.Sum() hashRes[HashModeNeedOnlySelf] = hex.EncodeToString(curSum[:]) - imps := make([]*packages.Package, 0, len(pkg.Imports)) - for _, imp := range pkg.Imports { - imps = append(imps, imp) - } - sort.Slice(imps, func(i, j int) bool { - return imps[i].PkgPath < imps[j].PkgPath + imps := maps.Values(pkg.Imports) + + slices.SortFunc(imps, func(a, b *packages.Package) int { + return strings.Compare(a.PkgPath, b.PkgPath) }) - calcDepsHash := func(depMode HashMode) error { - for _, dep := range imps { - if dep.PkgPath == "unsafe" { - continue - } + if err := c.computeDepsHash(HashModeNeedOnlySelf, imps, key); err != nil { + return nil, err + } + + curSum = key.Sum() + hashRes[HashModeNeedDirectDeps] = hex.EncodeToString(curSum[:]) + + if err := c.computeDepsHash(HashModeNeedAllDeps, imps, key); err != nil { + return nil, err + } + + curSum = key.Sum() + hashRes[HashModeNeedAllDeps] = hex.EncodeToString(curSum[:]) - depHash, depErr := c.packageHash(dep, depMode) - if depErr != nil { - return fmt.Errorf("failed to calculate hash for dependency %s with mode %d: %w", dep.Name, depMode, depErr) - } + return hashRes, nil +} - fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, depHash) +func (c *Cache) computeDepsHash(depMode HashMode, imps []*packages.Package, key *cache.Hash) error { + for _, dep := range imps { + if dep.PkgPath == "unsafe" { + continue } - return nil + + depHash, err := c.packageHash(dep, depMode) + if err != nil { + return fmt.Errorf("failed to calculate hash for dependency %s with mode %d: %w", dep.Name, depMode, err) + } + + fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, depHash) } - if err := calcDepsHash(HashModeNeedOnlySelf); err != nil { - return "", err + return nil +} + +func (c *Cache) putBytes(actionID cache.ActionID, buf *bytes.Buffer) error { + c.ioSem <- struct{}{} + + err := c.sw.TrackStageErr("cache io", func() error { + return cache.PutBytes(c.lowLevelCache, actionID, buf.Bytes()) + }) + + <-c.ioSem + + if err != nil { + return err } - curSum = key.Sum() - hashRes[HashModeNeedDirectDeps] = hex.EncodeToString(curSum[:]) + return nil +} - if err := calcDepsHash(HashModeNeedAllDeps); err != nil { - return "", err +func (c *Cache) getBytes(actionID cache.ActionID) ([]byte, error) { + c.ioSem <- struct{}{} + + cachedData, err := timeutils.TrackStage(c.sw, "cache io", func() ([]byte, error) { + b, _, errGB := cache.GetBytes(c.lowLevelCache, actionID) + return b, errGB + }) + + <-c.ioSem + + if err != nil { + return nil, err } - curSum = key.Sum() - hashRes[HashModeNeedAllDeps] = hex.EncodeToString(curSum[:]) - if _, ok := hashRes[mode]; !ok { - return "", fmt.Errorf("invalid mode %d", mode) + return cachedData, nil +} + +func (c *Cache) fileHash(f string) ([cache.HashSize]byte, error) { + c.ioSem <- struct{}{} + + h, err := cache.FileHash(f) + + <-c.ioSem + + if err != nil { + return [cache.HashSize]byte{}, err } - c.pkgHashes.Store(pkg, hashRes) - return hashRes[mode], nil + return h, nil +} + +func (c *Cache) encode(data any) (*bytes.Buffer, error) { + buf := &bytes.Buffer{} + err := c.sw.TrackStageErr("gob", func() error { + return gob.NewEncoder(buf).Encode(data) + }) + if err != nil { + return nil, fmt.Errorf("failed to gob encode: %w", err) + } + + return buf, nil +} + +func (c *Cache) decode(b []byte, data any) error { + err := c.sw.TrackStageErr("gob", func() error { + return gob.NewDecoder(bytes.NewReader(b)).Decode(data) + }) + if err != nil { + return fmt.Errorf("failed to gob decode: %w", err) + } + + return nil } func SetSalt(b *bytes.Buffer) { diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 000000000000..616e6344a281 --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,158 @@ +package cache + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/tools/go/packages" + + "github.com/golangci/golangci-lint/pkg/logutils" + "github.com/golangci/golangci-lint/pkg/timeutils" +) + +func setupCache(t *testing.T) *Cache { + t.Helper() + + log := logutils.NewStderrLog("skip") + sw := timeutils.NewStopwatch("pkgcache", log) + + pkgCache, err := NewCache(sw, log) + require.NoError(t, err) + + return pkgCache +} + +func fakePackage() *packages.Package { + return &packages.Package{ + PkgPath: "github.com/golangci/example", + CompiledGoFiles: []string{ + "./testdata/hello.go", + }, + Imports: map[string]*packages.Package{ + "a": { + PkgPath: "github.com/golangci/example/a", + }, + "b": { + PkgPath: "github.com/golangci/example/b", + }, + "unsafe": { + PkgPath: "unsafe", + }, + }, + } +} + +type Foo struct { + Value string +} + +func TestCache_Put(t *testing.T) { + t.Setenv("GOLANGCI_LINT_CACHE", t.TempDir()) + + pkgCache := setupCache(t) + + pkg := fakePackage() + + in := &Foo{Value: "hello"} + + err := pkgCache.Put(pkg, HashModeNeedAllDeps, "key", in) + require.NoError(t, err) + + out := &Foo{} + err = pkgCache.Get(pkg, HashModeNeedAllDeps, "key", out) + require.NoError(t, err) + + assert.Equal(t, in, out) + + pkgCache.Close() +} + +func TestCache_Get_missing_data(t *testing.T) { + t.Setenv("GOLANGCI_LINT_CACHE", t.TempDir()) + + pkgCache := setupCache(t) + + pkg := fakePackage() + + out := &Foo{} + err := pkgCache.Get(pkg, HashModeNeedAllDeps, "key", out) + require.Error(t, err) + + require.ErrorIs(t, err, ErrMissing) + + pkgCache.Close() +} + +func TestCache_buildKey(t *testing.T) { + pkgCache := setupCache(t) + + pkg := fakePackage() + + actionID, err := pkgCache.buildKey(pkg, HashModeNeedAllDeps, "") + require.NoError(t, err) + + assert.Equal(t, "f32bf1bf010aa9b570e081c64ec9e22e17aafa1e822990ba952905ec5fdf8d9d", fmt.Sprintf("%x", actionID)) +} + +func TestCache_pkgActionID(t *testing.T) { + pkgCache := setupCache(t) + + pkg := fakePackage() + + actionID, err := pkgCache.pkgActionID(pkg, HashModeNeedAllDeps) + require.NoError(t, err) + + assert.Equal(t, "f690f05acd1024386ae912d9ad9c04080523b9a899f6afe56ab3108d88215c1d", fmt.Sprintf("%x", actionID)) +} + +func TestCache_packageHash_load(t *testing.T) { + pkgCache := setupCache(t) + + pkg := fakePackage() + + pkgCache.pkgHashes.Store(pkg, hashResults{HashModeNeedAllDeps: "fake"}) + + hash, err := pkgCache.packageHash(pkg, HashModeNeedAllDeps) + require.NoError(t, err) + + assert.Equal(t, "fake", hash) +} + +func TestCache_packageHash_store(t *testing.T) { + pkgCache := setupCache(t) + + pkg := fakePackage() + + hash, err := pkgCache.packageHash(pkg, HashModeNeedAllDeps) + require.NoError(t, err) + + assert.Equal(t, "9c602ef861197b6807e82c99caa7c4042eb03c1a92886303fb02893744355131", hash) + + results, ok := pkgCache.pkgHashes.Load(pkg) + require.True(t, ok) + + hashRes := results.(hashResults) + + require.Len(t, hashRes, 3) + + assert.Equal(t, "8978e3d76c6f99e9663558d7147a7790f229a676804d1fde706a611898547b74", hashRes[HashModeNeedOnlySelf]) + assert.Equal(t, "b1aef902a0619b5cbfc2d6e2e91a73dd58dd448e58274b2d7a5ff8efd97aefa4", hashRes[HashModeNeedDirectDeps]) + assert.Equal(t, "9c602ef861197b6807e82c99caa7c4042eb03c1a92886303fb02893744355131", hashRes[HashModeNeedAllDeps]) +} + +func TestCache_computeHash(t *testing.T) { + pkgCache := setupCache(t) + + pkg := fakePackage() + + results, err := pkgCache.computePkgHash(pkg) + require.NoError(t, err) + + require.Len(t, results, 3) + + assert.Equal(t, "8978e3d76c6f99e9663558d7147a7790f229a676804d1fde706a611898547b74", results[HashModeNeedOnlySelf]) + assert.Equal(t, "b1aef902a0619b5cbfc2d6e2e91a73dd58dd448e58274b2d7a5ff8efd97aefa4", results[HashModeNeedDirectDeps]) + assert.Equal(t, "9c602ef861197b6807e82c99caa7c4042eb03c1a92886303fb02893744355131", results[HashModeNeedAllDeps]) +} diff --git a/internal/cache/testdata/hello.go b/internal/cache/testdata/hello.go new file mode 100644 index 000000000000..2e65d51de9cb --- /dev/null +++ b/internal/cache/testdata/hello.go @@ -0,0 +1,7 @@ +package testdata + +import "fmt" + +func Hello() { + fmt.Println("hello world") +} diff --git a/pkg/timeutils/stopwatch.go b/pkg/timeutils/stopwatch.go index d944dea2ea0e..95b16de9fc70 100644 --- a/pkg/timeutils/stopwatch.go +++ b/pkg/timeutils/stopwatch.go @@ -114,3 +114,25 @@ func (s *Stopwatch) TrackStage(name string, f func()) { s.stages[name] += time.Since(startedAt) s.mu.Unlock() } + +func (s *Stopwatch) TrackStageErr(name string, f func() error) error { + startedAt := time.Now() + err := f() + + s.mu.Lock() + s.stages[name] += time.Since(startedAt) + s.mu.Unlock() + + return err +} + +func TrackStage[T any](s *Stopwatch, name string, f func() (T, error)) (T, error) { + var result T + var err error + + s.TrackStage(name, func() { + result, err = f() + }) + + return result, err +}