From a94e494d53eb7f0e0e145619d8b6933208ad80e6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 25 May 2024 17:18:19 +0800 Subject: [PATCH 01/10] Refactor iterate git tree --- modules/actions/workflows.go | 3 +- modules/git/parse_nogogit.go | 23 ++++-- modules/git/repo_language_stats_nogogit.go | 38 +++++----- modules/git/tree_nogogit.go | 83 +++++++++++++++++----- routers/web/repo/treelist.go | 2 +- services/repository/files/tree.go | 2 +- 6 files changed, 105 insertions(+), 46 deletions(-) diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 0d2b0dd9194d9..1edf8498aad83 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -5,6 +5,7 @@ package actions import ( "bytes" + "context" "io" "strings" @@ -55,7 +56,7 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { return nil, err } - entries, err := tree.ListEntriesRecursiveFast() + entries, err := tree.ListEntriesRecursiveFast(context.Background()) if err != nil { return nil, err } diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index 546b38be37964..619d86a72a205 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -24,8 +24,15 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { var sepSpace = []byte{' '} func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { - var err error entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1) + return entries, iterateTreeEntries(data, ptree, func(entry *TreeEntry) error { + entries = append(entries, entry) + return nil + }) +} + +func iterateTreeEntries(data []byte, ptree *Tree, f func(entry *TreeEntry) error) error { + var err error for pos := 0; pos < len(data); { // expect line to be of the form: // \t @@ -39,7 +46,7 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { line := data[pos:posEnd] posTab := bytes.IndexByte(line, '\t') if posTab == -1 { - return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) + return fmt.Errorf("invalid ls-tree output (no tab): %q", line) } entry := new(TreeEntry) @@ -69,27 +76,29 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons entry.entryMode = EntryModeTree default: - return nil, fmt.Errorf("unknown type: %v", string(entryMode)) + return fmt.Errorf("unknown type: %v", string(entryMode)) } entry.ID, err = NewIDFromString(string(entryObjectID)) if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) + return fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) } if len(entryName) > 0 && entryName[0] == '"' { entry.name, err = strconv.Unquote(string(entryName)) if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) + return fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) } } else { entry.name = string(entryName) } pos = posEnd + 1 - entries = append(entries, entry) + if err := f(entry); err != nil { + return err + } } - return entries, nil + return nil } func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.Reader, sz int64) ([]*TreeEntry, error) { diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index de7707bd6cd8b..239ee344fd532 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -57,11 +57,6 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err tree := commit.Tree - entries, err := tree.ListEntriesRecursiveWithSize() - if err != nil { - return nil, err - } - checker, deferable := repo.CheckAttributeReader(commitID) defer deferable() @@ -77,10 +72,10 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguage := "" firstExcludedLanguageSize := int64(0) - for _, f := range entries { + if err := tree.IterateEntriesWithSize(func(f *TreeEntry) error { select { case <-repo.Ctx.Done(): - return sizes, repo.Ctx.Err() + return repo.Ctx.Err() default: } @@ -88,7 +83,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err content = contentBuf.Bytes() if f.Size() == 0 { - continue + return nil } isVendored := optional.None[bool]() @@ -101,22 +96,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err == nil { isVendored = AttributeToBool(attrs, AttributeLinguistVendored) if isVendored.ValueOrDefault(false) { - continue + return nil } isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) if isGenerated.ValueOrDefault(false) { - continue + return nil } isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) if isDocumentation.ValueOrDefault(false) { - continue + return nil } isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) if !isDetectable.ValueOrDefault(true) { - continue + return nil } hasLanguage := TryReadLanguageAttribute(attrs) @@ -131,7 +126,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err // this language will always be added to the size sizes[language] += f.Size() - continue + return nil } } } @@ -140,19 +135,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err enry.IsDotFile(f.Name()) || (!isDocumentation.Has() && enry.IsDocumentation(f.Name())) || enry.IsConfiguration(f.Name()) { - continue + return nil } // If content can not be read or file is too big just do detection by filename if f.Size() <= bigFileSize { if err := writeID(f.ID.String()); err != nil { - return nil, err + return err } _, _, size, err := ReadBatchLine(batchReader) if err != nil { log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) - return nil, err + return err } sizeToRead := size @@ -164,22 +159,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err _, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead)) if err != nil { - return nil, err + return err } content = contentBuf.Bytes() if err := DiscardFull(batchReader, discard); err != nil { - return nil, err + return err } } if !isGenerated.Has() && enry.IsGenerated(f.Name(), content) { - continue + return nil } // FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary? // - eg. do the all the detection tests using filename first before reading content. language := analyze.GetCodeLanguage(f.Name(), content) if language == "" { - continue + return nil } // group languages, such as Pug -> HTML; SCSS -> CSS @@ -200,6 +195,9 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguage = language firstExcludedLanguageSize += f.Size() } + return nil + }); err != nil { + return sizes, err } // If there are no included languages add the first excluded language diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 993b98edc2994..b7b61a934406e 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -6,6 +6,8 @@ package git import ( + "bufio" + "context" "io" "strings" ) @@ -91,34 +93,83 @@ func (t *Tree) ListEntries() (Entries, error) { // listEntriesRecursive returns all entries of current tree recursively including all subtrees // extraArgs could be "-l" to get the size, which is slower -func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { +func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArgs) (Entries, error) { if t.entriesRecursiveParsed { return t.entriesRecursive, nil } - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). - AddArguments(extraArgs...). - AddDynamicArguments(t.ID.String()). - RunStdBytes(&RunOpts{Dir: t.repo.Path}) - if runErr != nil { - return nil, runErr - } - - var err error - t.entriesRecursive, err = parseTreeEntries(stdout, t) + t.entriesRecursive = make([]*TreeEntry, 0) + err := t.iterateEntriesRecursive(func(entry *TreeEntry) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + t.entriesRecursive = append(t.entriesRecursive, entry) + return nil + }, extraArgs) if err == nil { t.entriesRecursiveParsed = true } - return t.entriesRecursive, err } // ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size -func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { - return t.listEntriesRecursive(nil) +func (t *Tree) ListEntriesRecursiveFast(ctx context.Context) (Entries, error) { + return t.listEntriesRecursive(ctx, nil) } // ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size -func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { - return t.listEntriesRecursive(TrustedCmdArgs{"--long"}) +func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error) { + return t.listEntriesRecursive(ctx, TrustedCmdArgs{"--long"}) +} + +// iterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees +// extraArgs could be "-l" to get the size, which is slower +func (t *Tree) iterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { + if t.entriesRecursiveParsed { + return nil + } + + reader, writer := io.Pipe() + done := make(chan error) + + go func(done chan error, writer *io.PipeWriter, reader *io.PipeReader) { + runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). + AddArguments(extraArgs...). + AddDynamicArguments(t.ID.String()). + Run(&RunOpts{ + Dir: t.repo.Path, + Stdout: writer, + }) + + _ = writer.Close() + _ = reader.Close() + + done <- runErr + }(done, writer, reader) + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + if err := scanner.Err(); err != nil { + return err + } + data := scanner.Bytes() + if err := iterateTreeEntries(data, t, func(entry *TreeEntry) error { + select { + case runErr := <-done: + return runErr + default: + return f(entry) + } + }); err != nil { + return err + } + } + t.entriesRecursiveParsed = true + return nil +} + +func (t *Tree) IterateEntriesWithSize(f func(*TreeEntry) error) error { + return t.iterateEntriesRecursive(f, TrustedCmdArgs{"--long"}) } diff --git a/routers/web/repo/treelist.go b/routers/web/repo/treelist.go index d11af4669f90c..3626081e2820b 100644 --- a/routers/web/repo/treelist.go +++ b/routers/web/repo/treelist.go @@ -21,7 +21,7 @@ func TreeList(ctx *context.Context) { return } - entries, err := tree.ListEntriesRecursiveFast() + entries, err := tree.ListEntriesRecursiveFast(ctx) if err != nil { ctx.ServerError("ListEntriesRecursiveFast", err) return diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go index 6775186afdd8e..9945fad417126 100644 --- a/services/repository/files/tree.go +++ b/services/repository/files/tree.go @@ -47,7 +47,7 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git tree.URL = repo.APIURL() + "/git/trees/" + url.PathEscape(tree.SHA) var entries git.Entries if recursive { - entries, err = gitTree.ListEntriesRecursiveWithSize() + entries, err = gitTree.ListEntriesRecursiveWithSize(ctx) } else { entries, err = gitTree.ListEntries() } From cd1c6657a9d040151487617f8c2bbb97ed8b648a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 25 May 2024 17:27:29 +0800 Subject: [PATCH 02/10] some fixes --- modules/git/tree_gogit.go | 7 ++++--- modules/git/tree_nogogit.go | 3 --- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index 421b0ecb0f0f9..bb32df4e2e19a 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -7,6 +7,7 @@ package git import ( + "context" "io" "github.com/go-git/go-git/v5/plumbing" @@ -57,7 +58,7 @@ func (t *Tree) ListEntries() (Entries, error) { } // ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees -func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { +func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error) { if t.gogitTree == nil { err := t.loadTreeObject() if err != nil { @@ -93,6 +94,6 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { } // ListEntriesRecursiveFast is the alias of ListEntriesRecursiveWithSize for the gogit version -func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { - return t.ListEntriesRecursiveWithSize() +func (t *Tree) ListEntriesRecursiveFast(ctx context.Context) (Entries, error) { + return t.ListEntriesRecursiveWithSize(ctx) } diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index b7b61a934406e..7eab4e4446a0d 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -108,9 +108,6 @@ func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArg t.entriesRecursive = append(t.entriesRecursive, entry) return nil }, extraArgs) - if err == nil { - t.entriesRecursiveParsed = true - } return t.entriesRecursive, err } From ae6370f8f1d6aa854113afbcec3f9ae7b7e9b797 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 25 May 2024 17:18:19 +0800 Subject: [PATCH 03/10] Refactor iterate git tree --- modules/git/tree_nogogit.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 7eab4e4446a0d..b7b61a934406e 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -108,6 +108,9 @@ func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArg t.entriesRecursive = append(t.entriesRecursive, entry) return nil }, extraArgs) + if err == nil { + t.entriesRecursiveParsed = true + } return t.entriesRecursive, err } From 293698d75d8a21f3d452cefc0551161565636004 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 25 May 2024 17:27:29 +0800 Subject: [PATCH 04/10] some fixes --- modules/git/tree_nogogit.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index b7b61a934406e..7eab4e4446a0d 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -108,9 +108,6 @@ func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArg t.entriesRecursive = append(t.entriesRecursive, entry) return nil }, extraArgs) - if err == nil { - t.entriesRecursiveParsed = true - } return t.entriesRecursive, err } From 7bdf6869a7f6575dad4cace70631084df34fa0a8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 29 Dec 2024 17:49:16 -0800 Subject: [PATCH 05/10] improve the code --- modules/git/tree_nogogit.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 7eab4e4446a0d..9b12a557d6d4e 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -99,16 +99,16 @@ func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArg } t.entriesRecursive = make([]*TreeEntry, 0) - err := t.iterateEntriesRecursive(func(entry *TreeEntry) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } + if err := t.iterateEntriesRecursive(ctx, func(entry *TreeEntry) error { t.entriesRecursive = append(t.entriesRecursive, entry) return nil - }, extraArgs) - return t.entriesRecursive, err + }, extraArgs); err != nil { + t.entriesRecursive = nil + return nil, err + } + + t.entriesRecursiveParsed = true + return t.entriesRecursive, nil } // ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size @@ -123,11 +123,7 @@ func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error // iterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees // extraArgs could be "-l" to get the size, which is slower -func (t *Tree) iterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { - if t.entriesRecursiveParsed { - return nil - } - +func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { reader, writer := io.Pipe() done := make(chan error) @@ -151,9 +147,12 @@ func (t *Tree) iterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs if err := scanner.Err(); err != nil { return err } + data := scanner.Bytes() if err := iterateTreeEntries(data, t, func(entry *TreeEntry) error { select { + case <-ctx.Done(): + return ctx.Err() case runErr := <-done: return runErr default: @@ -163,10 +162,9 @@ func (t *Tree) iterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs return err } } - t.entriesRecursiveParsed = true return nil } func (t *Tree) IterateEntriesWithSize(f func(*TreeEntry) error) error { - return t.iterateEntriesRecursive(f, TrustedCmdArgs{"--long"}) + return t.iterateEntriesRecursive(context.Background(), f, TrustedCmdArgs{"--long"}) } From 6325a14e08b8f4bab49dda4bd924d87fe262bf6f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 29 Dec 2024 18:45:17 -0800 Subject: [PATCH 06/10] Fix bug --- modules/git/repo_language_stats_nogogit.go | 6 ------ modules/git/tree_nogogit.go | 13 +++++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index 239ee344fd532..f7e27e414fcb1 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -73,12 +73,6 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguageSize := int64(0) if err := tree.IterateEntriesWithSize(func(f *TreeEntry) error { - select { - case <-repo.Ctx.Done(): - return repo.Ctx.Err() - default: - } - contentBuf.Reset() content = contentBuf.Bytes() diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 9b12a557d6d4e..9f85ce7083cc0 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -127,7 +127,7 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn reader, writer := io.Pipe() done := make(chan error) - go func(done chan error, writer *io.PipeWriter, reader *io.PipeReader) { + go func(t *Tree, done chan error, writer *io.PipeWriter) { runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). AddArguments(extraArgs...). AddDynamicArguments(t.ID.String()). @@ -137,10 +137,9 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn }) _ = writer.Close() - _ = reader.Close() done <- runErr - }(done, writer, reader) + }(t, done, writer) scanner := bufio.NewScanner(reader) for scanner.Scan() { @@ -150,13 +149,15 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn data := scanner.Bytes() if err := iterateTreeEntries(data, t, func(entry *TreeEntry) error { + if err := f(entry); err != nil { + return err + } + select { case <-ctx.Done(): return ctx.Err() case runErr := <-done: return runErr - default: - return f(entry) } }); err != nil { return err @@ -166,5 +167,5 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn } func (t *Tree) IterateEntriesWithSize(f func(*TreeEntry) error) error { - return t.iterateEntriesRecursive(context.Background(), f, TrustedCmdArgs{"--long"}) + return t.iterateEntriesRecursive(t.repo.Ctx, f, TrustedCmdArgs{"--long"}) } From 3c4e4fd2a27e194761f366662dc1edf869387871 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 30 Dec 2024 22:34:05 -0800 Subject: [PATCH 07/10] Fix bug --- modules/git/repo_language_stats_nogogit.go | 2 +- modules/git/tree_nogogit.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index f7e27e414fcb1..c9ca838cb8657 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -72,7 +72,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguage := "" firstExcludedLanguageSize := int64(0) - if err := tree.IterateEntriesWithSize(func(f *TreeEntry) error { + if err := tree.IterateEntriesRecursiveWithSize(func(f *TreeEntry) error { contentBuf.Reset() content = contentBuf.Bytes() diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 9f85ce7083cc0..e6dd4b4a0df89 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -158,6 +158,8 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn return ctx.Err() case runErr := <-done: return runErr + default: + return nil } }); err != nil { return err @@ -166,6 +168,6 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn return nil } -func (t *Tree) IterateEntriesWithSize(f func(*TreeEntry) error) error { +func (t *Tree) IterateEntriesRecursiveWithSize(f func(*TreeEntry) error) error { return t.iterateEntriesRecursive(t.repo.Ctx, f, TrustedCmdArgs{"--long"}) } From 9faa109ef37c3a767f128694094866c17bcd620f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 31 Dec 2024 15:50:16 -0800 Subject: [PATCH 08/10] more improvements --- modules/actions/workflows.go | 13 ++++++------- modules/git/repo_language_stats_nogogit.go | 4 ++-- modules/git/tree_nogogit.go | 10 +++------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 1edf8498aad83..02925e1e8ad1e 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -56,17 +56,16 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { return nil, err } - entries, err := tree.ListEntriesRecursiveFast(context.Background()) - if err != nil { - return nil, err - } - - ret := make(git.Entries, 0, len(entries)) - for _, entry := range entries { + ret := make(git.Entries, 0, 5) + if err := tree.IterateEntriesRecursive(context.Background(), func(entry *git.TreeEntry) error { if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") { ret = append(ret, entry) } + return nil + }, nil); err != nil { + return nil, err } + return ret, nil } diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index c9ca838cb8657..60183477b9499 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -72,7 +72,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguage := "" firstExcludedLanguageSize := int64(0) - if err := tree.IterateEntriesRecursiveWithSize(func(f *TreeEntry) error { + if err := tree.IterateEntriesRecursive(repo.Ctx, func(f *TreeEntry) error { contentBuf.Reset() content = contentBuf.Bytes() @@ -190,7 +190,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguageSize += f.Size() } return nil - }); err != nil { + }, TrustedCmdArgs{"--long"}); err != nil { // add --long to get size return sizes, err } diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index e6dd4b4a0df89..02e17eba56315 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -99,7 +99,7 @@ func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArg } t.entriesRecursive = make([]*TreeEntry, 0) - if err := t.iterateEntriesRecursive(ctx, func(entry *TreeEntry) error { + if err := t.IterateEntriesRecursive(ctx, func(entry *TreeEntry) error { t.entriesRecursive = append(t.entriesRecursive, entry) return nil }, extraArgs); err != nil { @@ -121,9 +121,9 @@ func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error return t.listEntriesRecursive(ctx, TrustedCmdArgs{"--long"}) } -// iterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees +// IterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees // extraArgs could be "-l" to get the size, which is slower -func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { +func (t *Tree) IterateEntriesRecursive(ctx context.Context, f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { reader, writer := io.Pipe() done := make(chan error) @@ -167,7 +167,3 @@ func (t *Tree) iterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn } return nil } - -func (t *Tree) IterateEntriesRecursiveWithSize(f func(*TreeEntry) error) error { - return t.iterateEntriesRecursive(t.repo.Ctx, f, TrustedCmdArgs{"--long"}) -} From f87a5941c6cbb2f0dcfb8c31bfbf685909bca072 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 31 Dec 2024 16:44:13 -0800 Subject: [PATCH 09/10] Some improvements --- modules/actions/workflows.go | 3 +- modules/git/repo_language_stats_nogogit.go | 2 +- modules/git/tree_gogit.go | 33 +++++++++++++++------- modules/git/tree_nogogit.go | 19 ++++++------- routers/web/repo/treelist.go | 2 +- services/repository/files/tree.go | 2 +- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 02925e1e8ad1e..28be0fbf1aeae 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -5,7 +5,6 @@ package actions import ( "bytes" - "context" "io" "strings" @@ -57,7 +56,7 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { } ret := make(git.Entries, 0, 5) - if err := tree.IterateEntriesRecursive(context.Background(), func(entry *git.TreeEntry) error { + if err := tree.IterateEntriesRecursive(func(entry *git.TreeEntry) error { if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") { ret = append(ret, entry) } diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index 60183477b9499..098f29707cf07 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -72,7 +72,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err firstExcludedLanguage := "" firstExcludedLanguageSize := int64(0) - if err := tree.IterateEntriesRecursive(repo.Ctx, func(f *TreeEntry) error { + if err := tree.IterateEntriesRecursive(func(f *TreeEntry) error { contentBuf.Reset() content = contentBuf.Bytes() diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index bb32df4e2e19a..fa8f19fadf71e 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -7,7 +7,6 @@ package git import ( - "context" "io" "github.com/go-git/go-git/v5/plumbing" @@ -58,7 +57,25 @@ func (t *Tree) ListEntries() (Entries, error) { } // ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees -func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error) { +func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { + var entries []*TreeEntry + if err := t.IterateEntriesRecursive(func(entry *TreeEntry) error { + entries = append(entries, convertedEntry) + return nil + }); err != nil { + return nil, err + } + return entries, nil +} + +// ListEntriesRecursiveFast is the alias of ListEntriesRecursiveWithSize for the gogit version +func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { + return t.ListEntriesRecursiveWithSize() +} + +// IterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees +// extraArgs could be "-l" to get the size, which is slower +func (t *Tree) IterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { if t.gogitTree == nil { err := t.loadTreeObject() if err != nil { @@ -66,7 +83,6 @@ func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error } } - var entries []*TreeEntry seen := map[plumbing.Hash]bool{} walker := object.NewTreeWalker(t.gogitTree, true, seen) for { @@ -87,13 +103,10 @@ func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error ptree: t, fullName: fullName, } - entries = append(entries, convertedEntry) + if err := f(convertedEntry); err != nil { + return nil, err + } } - return entries, nil -} - -// ListEntriesRecursiveFast is the alias of ListEntriesRecursiveWithSize for the gogit version -func (t *Tree) ListEntriesRecursiveFast(ctx context.Context) (Entries, error) { - return t.ListEntriesRecursiveWithSize(ctx) + return nil } diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 02e17eba56315..89454df047407 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -7,7 +7,6 @@ package git import ( "bufio" - "context" "io" "strings" ) @@ -93,13 +92,13 @@ func (t *Tree) ListEntries() (Entries, error) { // listEntriesRecursive returns all entries of current tree recursively including all subtrees // extraArgs could be "-l" to get the size, which is slower -func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArgs) (Entries, error) { +func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { if t.entriesRecursiveParsed { return t.entriesRecursive, nil } t.entriesRecursive = make([]*TreeEntry, 0) - if err := t.IterateEntriesRecursive(ctx, func(entry *TreeEntry) error { + if err := t.IterateEntriesRecursive(func(entry *TreeEntry) error { t.entriesRecursive = append(t.entriesRecursive, entry) return nil }, extraArgs); err != nil { @@ -112,18 +111,18 @@ func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArg } // ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size -func (t *Tree) ListEntriesRecursiveFast(ctx context.Context) (Entries, error) { - return t.listEntriesRecursive(ctx, nil) +func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { + return t.listEntriesRecursive(nil) } // ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size -func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error) { - return t.listEntriesRecursive(ctx, TrustedCmdArgs{"--long"}) +func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { + return t.listEntriesRecursive(TrustedCmdArgs{"--long"}) } // IterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees // extraArgs could be "-l" to get the size, which is slower -func (t *Tree) IterateEntriesRecursive(ctx context.Context, f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { +func (t *Tree) IterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error { reader, writer := io.Pipe() done := make(chan error) @@ -154,8 +153,8 @@ func (t *Tree) IterateEntriesRecursive(ctx context.Context, f func(entry *TreeEn } select { - case <-ctx.Done(): - return ctx.Err() + case <-t.repo.Ctx.Done(): + return t.repo.Ctx.Err() case runErr := <-done: return runErr default: diff --git a/routers/web/repo/treelist.go b/routers/web/repo/treelist.go index 3626081e2820b..d11af4669f90c 100644 --- a/routers/web/repo/treelist.go +++ b/routers/web/repo/treelist.go @@ -21,7 +21,7 @@ func TreeList(ctx *context.Context) { return } - entries, err := tree.ListEntriesRecursiveFast(ctx) + entries, err := tree.ListEntriesRecursiveFast() if err != nil { ctx.ServerError("ListEntriesRecursiveFast", err) return diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go index 9945fad417126..6775186afdd8e 100644 --- a/services/repository/files/tree.go +++ b/services/repository/files/tree.go @@ -47,7 +47,7 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git tree.URL = repo.APIURL() + "/git/trees/" + url.PathEscape(tree.SHA) var entries git.Entries if recursive { - entries, err = gitTree.ListEntriesRecursiveWithSize(ctx) + entries, err = gitTree.ListEntriesRecursiveWithSize() } else { entries, err = gitTree.ListEntries() } From e20a8654714c85ce9c87e8f0477586544c2f4e13 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 5 Jan 2025 23:02:10 -0800 Subject: [PATCH 10/10] Fix lint --- modules/git/tree_gogit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index fa8f19fadf71e..59f6129b2abb9 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -62,7 +62,7 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { if err := t.IterateEntriesRecursive(func(entry *TreeEntry) error { entries = append(entries, convertedEntry) return nil - }); err != nil { + }, nil); err != nil { return nil, err } return entries, nil