From 786965e00e9cc6cbb0af17e8dd85b5e4bfc42182 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 13 Jun 2019 22:06:26 +0100 Subject: [PATCH 01/19] Add basic repository lfs management --- models/lfs.go | 36 +++++- modules/lfs/server.go | 2 +- modules/repofiles/update.go | 2 +- modules/repofiles/upload.go | 2 +- options/locale/locale_en-US.ini | 4 + routers/repo/lfs.go | 169 ++++++++++++++++++++++++++ routers/routes/routes.go | 7 ++ templates/repo/settings/lfs.tmpl | 85 +++++++++++++ templates/repo/settings/lfs_file.tmpl | 66 ++++++++++ templates/repo/settings/navbar.tmpl | 5 + 10 files changed, 370 insertions(+), 8 deletions(-) create mode 100644 routers/repo/lfs.go create mode 100644 templates/repo/settings/lfs.tmpl create mode 100644 templates/repo/settings/lfs_file.tmpl diff --git a/models/lfs.go b/models/lfs.go index 94d3f579056e7..29f27f09f029d 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -106,21 +106,47 @@ func (repo *Repository) GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error // RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. -func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) error { +func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) (int64, error) { if len(oid) == 0 { - return ErrLFSObjectNotExist + return 0, ErrLFSObjectNotExist } sess := x.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { - return err + return -1, err } m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} if _, err := sess.Delete(m); err != nil { - return err + return -1, err } - return sess.Commit() + count, err := sess.Count(&LFSMetaObject{Oid: oid}) + if err != nil { + return count, err + } + + return count, sess.Commit() +} + +// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository +func (repo *Repository) GetLFSMetaObjects(page, pageSize int) ([]*LFSMetaObject, error) { + sess := x.NewSession() + defer sess.Close() + + if page >= 0 && pageSize > 0 { + start := 0 + if page > 0 { + start = (page - 1) * pageSize + } + sess.Limit(pageSize, start) + } + lfsObjects := make([]*LFSMetaObject, 0, pageSize) + return lfsObjects, sess.Find(&lfsObjects, &LFSMetaObject{RepositoryID: repo.ID}) +} + +// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository +func (repo *Repository) CountLFSMetaObjects() (int64, error) { + return x.Count(&LFSMetaObject{RepositoryID: repo.ID}) } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index bf5355acfc3b1..0bfaa5f35c972 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -330,7 +330,7 @@ func PutHandler(ctx *context.Context) { if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { ctx.Resp.WriteHeader(500) fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) - if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil { + if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil { log.Error("RemoveLFSMetaObjectByOid: %v", err) } return diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index f011017a5e87a..ff4b24adb34b7 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -374,7 +374,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} if !contentStore.Exists(lfsMetaObject) { if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { - if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { + if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) } return nil, err diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 2da101c64dad6..d626ab913ba1f 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -37,7 +37,7 @@ func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, orig continue } if !info.lfsMetaObject.Existing { - if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { + if _, err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { original = fmt.Errorf("%v, %v", original, err) } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 71c76fd9b6ada..36e4339740d99 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1316,6 +1316,10 @@ settings.unarchive.text = Un-Archiving the repo will restore its ability to rece settings.unarchive.success = The repo was successfully un-archived. settings.unarchive.error = An error occurred while trying to un-archive the repo. See the log for more details. settings.update_avatar_success = The repository avatar has been updated. +settings.lfs=LFS +settings.lfs_filelist=LFS files stored in this repository +settings.lfs_delete=Delete LFS file with OID %s +settings.lfs_delete_warning=Deleting an LFS file may cause object does not exist errors on checkout. Are you sure? diff.browse_source = Browse Source diff.parent = parent diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go new file mode 100644 index 0000000000000..f5cf13ce2a3e2 --- /dev/null +++ b/routers/repo/lfs.go @@ -0,0 +1,169 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + "bytes" + "fmt" + gotemplate "html/template" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" +) + +const ( + tplSettingsLFS base.TplName = "repo/settings/lfs" + tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" +) + +// LFSFiles shows a repository's LFS files +func LFSFiles(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFiles", nil) + return + } + page := ctx.QueryInt("page") + if page <= 1 { + page = 1 + } + total, err := ctx.Repo.Repository.CountLFSMetaObjects() + if err != nil { + ctx.ServerError("LFSFiles", err) + return + } + + pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5) + ctx.Data["Title"] = ctx.Tr("repo.settings.lfs") + ctx.Data["PageIsSettingsLFS"] = true + lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum) + if err != nil { + ctx.ServerError("LFSFiles", err) + return + } + ctx.Data["LFSFiles"] = lfsMetaObjects + ctx.Data["Page"] = pager + ctx.HTML(200, tplSettingsLFS) +} + +// LFSFileGet serves a single LFS file +func LFSFileGet(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFileGet", nil) + return + } + ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" + oid := ctx.Params("oid") + ctx.Data["Title"] = oid + ctx.Data["PageIsSettingsLFS"] = true + meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid) + if err != nil { + ctx.ServerError("LFSFileGet", err) + return + } + ctx.Data["LFSFile"] = meta + dataRc, err := lfs.ReadMetaObject(meta) + if err != nil { + ctx.ServerError("LFSFileGet", err) + return + } + defer dataRc.Close() + buf := make([]byte, 1024) + n, err := dataRc.Read(buf) + if err != nil { + ctx.ServerError("Data", err) + return + } + buf = buf[:n] + + isTextFile := base.IsTextFile(buf) + ctx.Data["IsTextFile"] = isTextFile + + fileSize := meta.Size + ctx.Data["FileSize"] = meta.Size + ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct") + switch { + case isTextFile: + if fileSize >= setting.UI.MaxDisplayFileSize { + ctx.Data["IsFileTooLarge"] = true + break + } + + d, _ := ioutil.ReadAll(dataRc) + buf = templates.ToUTF8WithFallback(append(buf, d...)) + + // Building code view blocks with line number on server side. + var fileContent string + if content, err := templates.ToUTF8WithErr(buf); err != nil { + log.Error("ToUTF8WithErr: %v", err) + fileContent = string(buf) + } else { + fileContent = content + } + + var output bytes.Buffer + lines := strings.Split(fileContent, "\n") + //Remove blank line at the end of file + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + for index, line := range lines { + line = gotemplate.HTMLEscapeString(line) + if index != len(lines)-1 { + line += "\n" + } + output.WriteString(fmt.Sprintf(`
  • %s
    • `, index+1, index+1, line)) + } + ctx.Data["FileContent"] = gotemplate.HTML(output.String()) + + output.Reset() + for i := 0; i < len(lines); i++ { + output.WriteString(fmt.Sprintf(`%d`, i+1, i+1)) + } + ctx.Data["LineNums"] = gotemplate.HTML(output.String()) + + case base.IsPDFFile(buf): + ctx.Data["IsPDFFile"] = true + case base.IsVideoFile(buf): + ctx.Data["IsVideoFile"] = true + case base.IsAudioFile(buf): + ctx.Data["IsAudioFile"] = true + case base.IsImageFile(buf): + ctx.Data["IsImageFile"] = true + } + ctx.HTML(200, tplSettingsLFSFile) +} + +// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it +func LFSDelete(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFileGet", nil) + return + } + oid := ctx.Params("oid") + count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid) + if err != nil { + ctx.ServerError("LFSDelete", err) + return + } + // FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here + // Please note a similar condition happens in models/repo.go DeleteRepository + if count == 0 { + oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:]) + err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath)) + if err != nil { + ctx.ServerError("LFSDelete", err) + return + } + } + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 744088a9d7b1e..900db39804d40 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -662,8 +662,15 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/delete", repo.DeleteDeployKey) }) + m.Group("/lfs", func() { + m.Get("", repo.LFSFiles) + m.Get("/show/:oid", repo.LFSFileGet) + m.Post("/delete/:oid", repo.LFSDelete) + }) + }, func(ctx *context.Context) { ctx.Data["PageIsSettings"] = true + ctx.Data["LFSStartServer"] = setting.LFS.StartServer }) }, reqSignIn, context.RepoAssignment(), reqRepoAdmin, context.UnitTypes(), context.RepoRef()) diff --git a/templates/repo/settings/lfs.tmpl b/templates/repo/settings/lfs.tmpl new file mode 100644 index 0000000000000..aebf619cd0408 --- /dev/null +++ b/templates/repo/settings/lfs.tmpl @@ -0,0 +1,85 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} + {{template "repo/settings/navbar" .}} +
    + {{template "base/alert" .}} + + + + + + + + {{range .LFSFiles}} + + + + + + + {{end}} + +
    {{.i18n.Tr "repo.settings.lfs_filelist"}}
    + + + {{.Oid}} + + + {{FileSize .Size}}{{TimeSince .CreatedUnix.AsTime $.Lang}} + +
    + {{template "base/paginate" .}} + {{range .LFSFiles}} + + {{end}} +
    + +
    + +
    +
    + +{{template "base/footer" .}} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl new file mode 100644 index 0000000000000..25adaba332faa --- /dev/null +++ b/templates/repo/settings/lfs_file.tmpl @@ -0,0 +1,66 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} + {{template "repo/settings/navbar" .}} +
    + {{template "base/alert" .}} +
    +

    + {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}} +

    +
    +
    + {{if .IsMarkup}} + {{if .FileContent}}{{.FileContent | Safe}}{{end}} + {{else if .IsRenderedHTML}} + 
    {{if .FileContent}}{{.FileContent | Str2html}}{{end}}
    + {{else if not .IsTextFile}} +
    + {{if .IsImageFile}} + + {{else if .IsVideoFile}} + + {{else if .IsAudioFile}} + + {{else if .IsPDFFile}} + + {{else}} + {{.i18n.Tr "repo.file_view_raw"}} + {{end}} +
    + {{else if .FileSize}} + + + + {{if .IsFileTooLarge}} + + {{else}} + + + {{end}} + + +
    {{.i18n.Tr "repo.file_too_large"}}{{.LineNums}}
      {{.FileContent}}
    + {{end}} +
    +
    +
    +
    +
    + +{{template "base/footer" .}} diff --git a/templates/repo/settings/navbar.tmpl b/templates/repo/settings/navbar.tmpl index 24082000e2ae9..abd6e285dc3e3 100644 --- a/templates/repo/settings/navbar.tmpl +++ b/templates/repo/settings/navbar.tmpl @@ -21,4 +21,9 @@ {{.i18n.Tr "repo.settings.deploy_keys"}} + {{if .LFSStartServer}} + + {{.i18n.Tr "repo.settings.lfs"}} + + {{end}} From c3c2ba38ead2742a76ab71f7627caba427d9fa5e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 14 Jun 2019 17:18:12 +0100 Subject: [PATCH 02/19] add auto-associate function --- models/lfs.go | 44 ++++ models/repo_list.go | 48 ++-- options/locale/locale_en-US.ini | 8 + public/css/index.css | 1 + public/less/_base.less | 10 + routers/repo/lfs.go | 253 +++++++++++++++++++++- routers/routes/routes.go | 2 + templates/repo/settings/lfs.tmpl | 34 +-- templates/repo/settings/lfs_pointers.tmpl | 67 ++++++ 9 files changed, 418 insertions(+), 49 deletions(-) create mode 100644 templates/repo/settings/lfs_pointers.tmpl diff --git a/models/lfs.go b/models/lfs.go index 29f27f09f029d..ee9132265abe4 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -8,6 +8,7 @@ import ( "io" "code.gitea.io/gitea/modules/util" + "github.com/go-xorm/builder" ) // LFSMetaObject stores metadata for LFS tracked files. @@ -150,3 +151,46 @@ func (repo *Repository) GetLFSMetaObjects(page, pageSize int) ([]*LFSMetaObject, func (repo *Repository) CountLFSMetaObjects() (int64, error) { return x.Count(&LFSMetaObject{RepositoryID: repo.ID}) } + +// LFSObjectAccessible checks if a provided Oid is accessible to the user +func LFSObjectAccessible(user *User, oid string) (bool, error) { + if user.IsAdmin { + count, err := x.Count(&LFSMetaObject{Oid: oid}) + return (count > 0), err + } + cond := accessibleRepositoryCondition(user.ID) + count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid}) + return (count > 0), err +} + +// LFSAutoAssociate auto associates accessible LFSMetaObjects +func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + oids := make([]interface{}, len(metas)) + for i, meta := range metas { + oids[i] = meta.Oid + } + + cond := builder.NewCond() + if !user.IsAdmin { + cond = builder.In("`lfs_meta_object`.repository_id", + builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user.ID))) + } + newMetas := make([]*LFSMetaObject, 0, len(metas)) + if err := sess.Cols("oid", "size").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { + return err + } + for i := range newMetas { + newMetas[i].RepositoryID = repoID + } + if _, err := sess.InsertMulti(newMetas); err != nil { + return err + } + + return sess.Commit() +} diff --git a/models/repo_list.go b/models/repo_list.go index 5655404f7c71f..f746e7d832bc0 100644 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -174,28 +174,7 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err if opts.Private { if !opts.UserIsAdmin && opts.UserID != 0 && opts.UserID != opts.OwnerID { // OK we're in the context of a User - // We should be Either - cond = cond.And(builder.Or( - // 1. Be able to see all non-private repositories that either: - cond.And( - builder.Eq{"is_private": false}, - builder.Or( - // A. Aren't in organisations __OR__ - builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})), - // B. Isn't a private organisation. (Limited is OK because we're logged in) - builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))), - ), - // 2. Be able to see all repositories that we have access to - builder.In("id", builder.Select("repo_id"). - From("`access`"). - Where(builder.And( - builder.Eq{"user_id": opts.UserID}, - builder.Gt{"mode": int(AccessModeNone)}))), - // 3. Be able to see all repositories that we are in a team - builder.In("id", builder.Select("`team_repo`.repo_id"). - From("team_repo"). - Where(builder.Eq{"`team_user`.uid": opts.UserID}). - Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id")))) + cond = cond.And(accessibleRepositoryCondition(opts.UserID)) } } else { // Not looking at private organisations @@ -307,6 +286,31 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err return repos, count, nil } +// accessibleRepositoryCondition takes a user a returns a condition for checking if a repository is accessible +func accessibleRepositoryCondition(userID int64) builder.Cond { + return builder.Or( + // 1. Be able to see all non-private repositories that either: + builder.And( + builder.Eq{"`repository`.is_private": false}, + builder.Or( + // A. Aren't in organisations __OR__ + builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})), + // B. Isn't a private organisation. (Limited is OK because we're logged in) + builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))), + ), + // 2. Be able to see all repositories that we have access to + builder.In("`repository`.id", builder.Select("repo_id"). + From("`access`"). + Where(builder.And( + builder.Eq{"user_id": userID}, + builder.Gt{"mode": int(AccessModeNone)}))), + // 3. Be able to see all repositories that we are in a team + builder.In("`repository`.id", builder.Select("`team_repo`.repo_id"). + From("team_repo"). + Where(builder.Eq{"`team_user`.uid": userID}). + Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id"))) +} + // FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) { var accessCond builder.Cond = builder.Eq{"is_private": false} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 36e4339740d99..fbb2ee0a6492c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1320,6 +1320,14 @@ settings.lfs=LFS settings.lfs_filelist=LFS files stored in this repository settings.lfs_delete=Delete LFS file with OID %s settings.lfs_delete_warning=Deleting an LFS file may cause object does not exist errors on checkout. Are you sure? +settings.lfs_findpointerfiles=Find Pointer files +settings.lfs_pointers.found=Found %d blob pointer(s) - %d associated, %d unassociated (%d missing from store) +settings.lfs_pointers.sha=SHA +settings.lfs_pointers.oid=OID +settings.lfs_pointers.inRepo=In Repo +settings.lfs_pointers.exists=Exists in store +settings.lfs_pointers.accessible=Accessible to User +settings.lfs_pointers.associateAccessible=Associate accessible %d OIDs diff.browse_source = Browse Source diff.parent = parent diff --git a/public/css/index.css b/public/css/index.css index d192f43d1529a..280687f89e746 100644 --- a/public/css/index.css +++ b/public/css/index.css @@ -125,6 +125,7 @@ code.wrap,pre.wrap{white-space:pre-wrap;word-break:break-all;overflow-wrap:break .ui .form .fake{display:none!important} .ui .form .sub.field{margin-left:25px} .ui .sha.label{font-family:'SF Mono',Consolas,Menlo,'Liberation Mono',Monaco,'Lucida Console',monospace;font-size:13px;padding:6px 10px 4px 10px;font-weight:400;margin:0 6px} +.ui .button.truncate{display:inline-block;max-width:100%;overflow:hidden;text-overflow:ellipsis;vertical-align:top;white-space:nowrap;margin-right:6px} .ui.status.buttons .octicon{margin-right:4px} .ui.inline.delete-button{padding:8px 15px;font-weight:400} .ui .background.red{background-color:#d95c5c!important} diff --git a/public/less/_base.less b/public/less/_base.less index 13ae1ad665778..aab040a00612f 100644 --- a/public/less/_base.less +++ b/public/less/_base.less @@ -533,6 +533,16 @@ code { margin: 0 6px; } + .button.truncate { + display: inline-block; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + vertical-align: top; + white-space: nowrap; + margin-right: 6px; + } + &.status.buttons { .octicon { margin-right: 4px; diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index f5cf13ce2a3e2..05cc0aa7c4a53 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -5,25 +5,34 @@ package repo import ( + "bufio" "bytes" "fmt" gotemplate "html/template" + "io" "io/ioutil" "os" "path/filepath" + "strconv" "strings" + "sync" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "github.com/Unknwon/com" + "github.com/mcuadros/go-version" ) const ( - tplSettingsLFS base.TplName = "repo/settings/lfs" - tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" + tplSettingsLFS base.TplName = "repo/settings/lfs" + tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" + tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers" ) // LFSFiles shows a repository's LFS files @@ -67,6 +76,10 @@ func LFSFileGet(ctx *context.Context) { ctx.Data["PageIsSettingsLFS"] = true meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid) if err != nil { + if err == models.ErrLFSObjectNotExist { + ctx.NotFound("LFSFileGet", nil) + return + } ctx.ServerError("LFSFileGet", err) return } @@ -167,3 +180,239 @@ func LFSDelete(ctx *context.Context) { } ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") } + +// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store +func LFSPointerFiles(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFileGet", nil) + return + } + ctx.Data["PageIsSettingsLFS"] = true + binVersion, err := git.BinVersion() + if err != nil { + log.Fatal("Error retrieving git version: %v", err) + } + + if !version.Compare(binVersion, "2.6.0", ">=") { + ctx.ServerError("LFSPointerFiles", fmt.Errorf("Git version too low")) + return + } + + basePath := ctx.Repo.Repository.RepoPath() + + pointerChan := make(chan pointerResult) + + catFileCheckReader, catFileCheckWriter := io.Pipe() + shasToBatchReader, shasToBatchWriter := io.Pipe() + catFileBatchReader, catFileBatchWriter := io.Pipe() + wg := sync.WaitGroup{} + wg.Add(5) + + var numPointers, numAssociated, numNoExist, numAssociatable int + + go func() { + defer wg.Done() + pointers := make([]pointerResult, 0, 50) + for pointer := range pointerChan { + pointers = append(pointers, pointer) + if pointer.InRepo { + numAssociated++ + } + if !pointer.Exists { + numNoExist++ + } + if !pointer.InRepo && pointer.Accessible { + numAssociatable++ + } + } + numPointers = len(pointers) + ctx.Data["Pointers"] = pointers + ctx.Data["NumPointers"] = numPointers + ctx.Data["NumAssociated"] = numAssociated + ctx.Data["NumAssociatable"] = numAssociatable + ctx.Data["NumNoExist"] = numNoExist + ctx.Data["NumNotAssociated"] = numPointers - numAssociated + }() + go readCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) + go doCatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) + go readCatFileBatchCheckAllObjects(catFileCheckReader, shasToBatchWriter, &wg) + go doCatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath) + + wg.Wait() + ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" + ctx.HTML(200, tplSettingsLFSPointers) +} + +type pointerResult struct { + SHA string + Oid string + Size int64 + InRepo bool + Exists bool + Accessible bool +} + +func doCatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer catFileCheckWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") + if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil { + _ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} + +func readCatFileBatchCheckAllObjects(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { + defer wg.Done() + defer catFileCheckReader.Close() + scanner := bufio.NewScanner(catFileCheckReader) + defer func() { + _ = shasToBatchWriter.CloseWithError(scanner.Err()) + }() + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 3 || fields[1] != "blob" { + continue + } + size, _ := strconv.Atoi(string(fields[2])) + if size > 1024 { + continue + } + toWrite := []byte(fields[0] + "\n") + for len(toWrite) > 0 { + n, err := shasToBatchWriter.Write(toWrite) + if err != nil { + _ = catFileCheckReader.CloseWithError(err) + break + } + toWrite = toWrite[n:] + } + } +} + +func doCatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer shasToBatchReader.Close() + defer catFileBatchWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil { + _ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} + +func readCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { + defer wg.Done() + defer catFileBatchReader.Close() + contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath} + + bufferedReader := bufio.NewReader(catFileBatchReader) + buf := make([]byte, 1025) + for { + // File descriptor line: sha + sha, err := bufferedReader.ReadString(' ') + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + // Throw away the blob + if _, err := bufferedReader.ReadString(' '); err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + sizeStr, err := bufferedReader.ReadString('\n') + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + pointerBuf := buf[:size+1] + if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + pointerBuf = pointerBuf[:size] + // Now we need to check if the pointerBuf is an LFS pointer + pointer := lfs.IsPointerFile(&pointerBuf) + if pointer == nil { + continue + } + + result := pointerResult{ + SHA: strings.TrimSpace(sha), + Oid: pointer.Oid, + Size: pointer.Size, + } + + // Then we need to check that this pointer is in the db + if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { + if err != models.ErrLFSObjectNotExist { + _ = catFileBatchReader.CloseWithError(err) + break + } + } else { + result.InRepo = true + } + + result.Exists = contentStore.Exists(pointer) + + if result.Exists { + if !result.InRepo { + // Can we fix? + // OK well that's "simple" + // - we need to check whether current user has access to a repo that has access to the file + result.Accessible, err = models.LFSObjectAccessible(user, result.Oid) + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + } else { + result.Accessible = true + } + } + pointerChan <- result + } + close(pointerChan) +} + +// LFSAutoAssociate auto associates accessible lfs files +func LFSAutoAssociate(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSAutoAssociate", nil) + return + } + oids := ctx.QueryStrings("oid") + metas := make([]*models.LFSMetaObject, len(oids)) + for i, oid := range oids { + idx := strings.IndexRune(oid, ' ') + if idx < 0 || idx+1 > len(oid) { + ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid)) + return + } + var err error + metas[i] = &models.LFSMetaObject{} + metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64() + if err != nil { + ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err)) + return + } + metas[i].Oid = oid[:idx] + //metas[i].RepositoryID = ctx.Repo.Repository.ID + } + if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil { + ctx.ServerError("LFSAutoAssociate", err) + return + } + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 900db39804d40..84dce8247cbce 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -666,6 +666,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", repo.LFSFiles) m.Get("/show/:oid", repo.LFSFileGet) m.Post("/delete/:oid", repo.LFSDelete) + m.Get("/pointers", repo.LFSPointerFiles) + m.Post("/pointers/associate", repo.LFSAutoAssociate) }) }, func(ctx *context.Context) { diff --git a/templates/repo/settings/lfs.tmpl b/templates/repo/settings/lfs.tmpl index aebf619cd0408..630b446027001 100644 --- a/templates/repo/settings/lfs.tmpl +++ b/templates/repo/settings/lfs.tmpl @@ -4,18 +4,19 @@ {{template "repo/settings/navbar" .}}
    {{template "base/alert" .}} - - - - - - +

    + {{.i18n.Tr "repo.settings.lfs_filelist"}} +
    + {{.i18n.Tr "repo.settings.lfs_findpointerfiles"}} +
    +

    +
    {{.i18n.Tr "repo.settings.lfs_filelist"}}
    {{range .LFSFiles}} - + + @@ -54,6 +55,9 @@ {{end}} + From 8ed7fefbe0fd8bc3bf0de497c873f532fd9630c8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 23 Jul 2019 10:54:25 +0100 Subject: [PATCH 11/19] Improve find commits functionality --- routers/repo/lfs.go | 45 ++++++++++++++++++----- templates/repo/settings/lfs_pointers.tmpl | 4 +- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index ad1015fb123e5..136e9427d9112 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -188,12 +189,19 @@ func LFSDelete(ctx *context.Context) { } type lfsResult struct { - Name string - SHA string - Summary string - When time.Time + Name string + SHA string + Summary string + When time.Time + ParentHashes []plumbing.Hash } +type lfsResultSlice []lfsResult + +func (a lfsResultSlice) Len() int { return len(a) } +func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } + // LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha func LFSFileFind(ctx *context.Context) { if !setting.LFS.StartServer { @@ -223,6 +231,7 @@ func LFSFileFind(ctx *context.Context) { ctx.Data["Size"] = size ctx.Data["SHA"] = sha + resultsMap := map[string]*lfsResult{} results := make([]lfsResult, 0) gogitRepo := ctx.Repo.GitRepo.GoGitRepo() @@ -250,17 +259,33 @@ func LFSFileFind(ctx *context.Context) { break } if entry.Hash == hash { - results = append(results, lfsResult{ - Name: name, - SHA: gitCommit.Hash.String(), - Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], - When: gitCommit.Author.When, - }) + result := lfsResult{ + Name: name, + SHA: gitCommit.Hash.String(), + Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], + When: gitCommit.Author.When, + ParentHashes: gitCommit.ParentHashes, + } + resultsMap[gitCommit.Hash.String()+":"+name] = &result } } return nil }) + for _, result := range resultsMap { + hasParent := false + for _, parentHash := range result.ParentHashes { + if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { + break + } + } + if !hasParent { + results = append(results, *result) + } + } + + sort.Sort(lfsResultSlice(results)) + ctx.Data["Results"] = results ctx.HTML(200, tplSettingsLFSFileFind) } diff --git a/templates/repo/settings/lfs_pointers.tmpl b/templates/repo/settings/lfs_pointers.tmpl index d8c0baadec2b9..1bd48de15780b 100644 --- a/templates/repo/settings/lfs_pointers.tmpl +++ b/templates/repo/settings/lfs_pointers.tmpl @@ -25,8 +25,8 @@ - - + + From 0f1309045e5b523f4df0ee903b7ae8d03adfa619 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 23 Jul 2019 10:58:37 +0100 Subject: [PATCH 12/19] take account of foreach error --- routers/repo/lfs.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 136e9427d9112..427d97849f122 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -246,7 +246,7 @@ func LFSFileFind(ctx *context.Context) { return } - commitsIter.ForEach(func(gitCommit *object.Commit) error { + err = commitsIter.ForEach(func(gitCommit *object.Commit) error { tree, err := gitCommit.Tree() if err != nil { return err @@ -271,6 +271,11 @@ func LFSFileFind(ctx *context.Context) { } return nil }) + if err != nil && err != io.EOF { + log.Error("Failure in CommitIter.ForEach: %v", err) + ctx.ServerError("LFSFind: IterateCommits ForEach", err) + return + } for _, result := range resultsMap { hasParent := false From ab980f51aea2cb6007e81b4c458831883059a9f3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 23 Jul 2019 11:51:58 +0100 Subject: [PATCH 13/19] remove code duplication and extract to pipeline --- modules/git/pipeline/catfile.go | 94 +++++++++++++++++++++++ modules/git/pipeline/revlist.go | 75 ++++++++++++++++++ modules/pull/lfs.go | 116 ++-------------------------- routers/repo/lfs.go | 132 +++----------------------------- 4 files changed, 186 insertions(+), 231 deletions(-) create mode 100644 modules/git/pipeline/catfile.go create mode 100644 modules/git/pipeline/revlist.go diff --git a/modules/git/pipeline/catfile.go b/modules/git/pipeline/catfile.go new file mode 100644 index 0000000000000..813c6a372e712 --- /dev/null +++ b/modules/git/pipeline/catfile.go @@ -0,0 +1,94 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pipeline + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" + "sync" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// CatFileBatchCheck runs cat-file with --batch-check +func CatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer shasToCheckReader.Close() + defer catFileCheckWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + cmd := git.NewCommand("cat-file", "--batch-check") + if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil { + _ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} + +// CatFileBatchCheckAllObjects runs cat-file with --batch-check --batch-all +func CatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) { + defer wg.Done() + defer catFileCheckWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") + if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil { + log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String()) + err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String()) + _ = catFileCheckWriter.CloseWithError(err) + errChan <- err + } +} + +// CatFileBatch runs cat-file --batch +func CatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer shasToBatchReader.Close() + defer catFileBatchWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil { + _ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} + +// BlobsLessThan1024FromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <1024 in size +func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { + defer wg.Done() + defer catFileCheckReader.Close() + scanner := bufio.NewScanner(catFileCheckReader) + defer func() { + _ = shasToBatchWriter.CloseWithError(scanner.Err()) + }() + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 3 || fields[1] != "blob" { + continue + } + size, _ := strconv.Atoi(string(fields[2])) + if size > 1024 { + continue + } + toWrite := []byte(fields[0] + "\n") + for len(toWrite) > 0 { + n, err := shasToBatchWriter.Write(toWrite) + if err != nil { + _ = catFileCheckReader.CloseWithError(err) + break + } + toWrite = toWrite[n:] + } + } +} diff --git a/modules/git/pipeline/revlist.go b/modules/git/pipeline/revlist.go new file mode 100644 index 0000000000000..4e13e19444cf4 --- /dev/null +++ b/modules/git/pipeline/revlist.go @@ -0,0 +1,75 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pipeline + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "sync" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// RevListAllObjects runs rev-list --objects --all and writes to a pipewriter +func RevListAllObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) { + defer wg.Done() + defer revListWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + cmd := git.NewCommand("rev-list", "--objects", "--all") + if err := cmd.RunInDirPipeline(basePath, revListWriter, stderr); err != nil { + log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String()) + err = fmt.Errorf("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String()) + _ = revListWriter.CloseWithError(err) + errChan <- err + } +} + +// RevListObjects run rev-list --objects from headSHA to baseSHA +func RevListObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) { + defer wg.Done() + defer revListWriter.Close() + stderr := new(bytes.Buffer) + var errbuf strings.Builder + cmd := git.NewCommand("rev-list", "--objects", headSHA, "--not", baseSHA) + if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil { + log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()) + errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()) + } +} + +// BlobsFromRevListObjects reads a RevListAllObjects and only selects blobs +func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) { + defer wg.Done() + defer revListReader.Close() + scanner := bufio.NewScanner(revListReader) + defer func() { + _ = shasToCheckWriter.CloseWithError(scanner.Err()) + }() + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 2 || len(fields[1]) == 0 { + continue + } + toWrite := []byte(fields[0] + "\n") + for len(toWrite) > 0 { + n, err := shasToCheckWriter.Write(toWrite) + if err != nil { + _ = revListReader.CloseWithError(err) + break + } + toWrite = toWrite[n:] + } + } +} diff --git a/modules/pull/lfs.go b/modules/pull/lfs.go index 77890667d63d1..a1981b8253690 100644 --- a/modules/pull/lfs.go +++ b/modules/pull/lfs.go @@ -7,15 +7,12 @@ package pull import ( "bufio" - "bytes" - "fmt" "io" "strconv" - "strings" "sync" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/pipeline" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" ) @@ -41,22 +38,22 @@ func LFSPush(tmpBasePath, mergeHeadSHA, mergeBaseSHA string, pr *models.PullRequ // 6. Take the output of cat-file --batch and check if each file in turn // to see if they're pointers to files in the LFS store associated with // the head repo and add them to the base repo if so - go readCatFileBatch(catFileBatchReader, &wg, pr) + go createLFSMetaObjectsFromCatFileBatch(catFileBatchReader, &wg, pr) // 5. Take the shas of the blobs and batch read them - go doCatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, tmpBasePath) + go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, tmpBasePath) // 4. From the provided objects restrict to blobs <=1k - go readCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) + go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) // 3. Run batch-check on the objects retrieved from rev-list - go doCatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, tmpBasePath) + go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, tmpBasePath) // 2. Check each object retrieved rejecting those without names as they will be commits or trees - go readRevListObjects(revListReader, shasToCheckWriter, &wg) + go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) // 1. Run rev-list objects from mergeHead to mergeBase - go doRevListObjects(revListWriter, &wg, tmpBasePath, mergeHeadSHA, mergeBaseSHA, errChan) + go pipeline.RevListObjects(revListWriter, &wg, tmpBasePath, mergeHeadSHA, mergeBaseSHA, errChan) wg.Wait() select { @@ -69,104 +66,7 @@ func LFSPush(tmpBasePath, mergeHeadSHA, mergeBaseSHA string, pr *models.PullRequ return nil } -func doRevListObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) { - defer wg.Done() - defer revListWriter.Close() - stderr := new(bytes.Buffer) - var errbuf strings.Builder - cmd := git.NewCommand("rev-list", "--objects", headSHA, "--not", baseSHA) - if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil { - log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()) - errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()) - } -} - -func readRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) { - defer wg.Done() - defer revListReader.Close() - defer shasToCheckWriter.Close() - scanner := bufio.NewScanner(revListReader) - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - fields := strings.Split(line, " ") - if len(fields) < 2 || len(fields[1]) == 0 { - continue - } - toWrite := []byte(fields[0] + "\n") - for len(toWrite) > 0 { - n, err := shasToCheckWriter.Write(toWrite) - if err != nil { - _ = revListReader.CloseWithError(err) - break - } - toWrite = toWrite[n:] - } - } - _ = shasToCheckWriter.CloseWithError(scanner.Err()) -} - -func doCatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { - defer wg.Done() - defer shasToCheckReader.Close() - defer catFileCheckWriter.Close() - - stderr := new(bytes.Buffer) - var errbuf strings.Builder - cmd := git.NewCommand("cat-file", "--batch-check") - if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil { - _ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String())) - } -} - -func readCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { - defer wg.Done() - defer catFileCheckReader.Close() - - scanner := bufio.NewScanner(catFileCheckReader) - defer func() { - _ = shasToBatchWriter.CloseWithError(scanner.Err()) - }() - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - fields := strings.Split(line, " ") - if len(fields) < 3 || fields[1] != "blob" { - continue - } - size, _ := strconv.Atoi(string(fields[2])) - if size > 1024 { - continue - } - toWrite := []byte(fields[0] + "\n") - for len(toWrite) > 0 { - n, err := shasToBatchWriter.Write(toWrite) - if err != nil { - _ = catFileCheckReader.CloseWithError(err) - break - } - toWrite = toWrite[n:] - } - } -} - -func doCatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { - defer wg.Done() - defer shasToBatchReader.Close() - defer catFileBatchWriter.Close() - - stderr := new(bytes.Buffer) - var errbuf strings.Builder - if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil { - _ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())) - } -} - -func readCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pr *models.PullRequest) { +func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pr *models.PullRequest) { defer wg.Done() defer catFileBatchReader.Close() diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 427d97849f122..e8ae46f03edfe 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/pipeline" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -344,18 +345,18 @@ func LFSPointerFiles(ctx *context.Context) { ctx.Data["NumNoExist"] = numNoExist ctx.Data["NumNotAssociated"] = numPointers - numAssociated }() - go readCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) - go doCatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) - go readCatFileBatchCheckAllObjects(catFileCheckReader, shasToBatchWriter, &wg) + go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) + go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) + go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) if !version.Compare(binVersion, "2.6.0", ">=") { revListReader, revListWriter := io.Pipe() shasToCheckReader, shasToCheckWriter := io.Pipe() wg.Add(2) - go doCatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) - go readRevListAllObjects(revListReader, shasToCheckWriter, &wg) - go doRevListAllObjects(revListWriter, &wg, basePath, errChan) + go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) + go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) + go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) } else { - go doCatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) + go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) } wg.Wait() @@ -378,122 +379,7 @@ type pointerResult struct { Accessible bool } -func doRevListAllObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) { - defer wg.Done() - defer revListWriter.Close() - - stderr := new(bytes.Buffer) - var errbuf strings.Builder - cmd := git.NewCommand("rev-list", "--objects", "--all") - if err := cmd.RunInDirPipeline(basePath, revListWriter, stderr); err != nil { - log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String()) - err = fmt.Errorf("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String()) - _ = revListWriter.CloseWithError(err) - errChan <- err - } -} - -func readRevListAllObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) { - defer wg.Done() - defer revListReader.Close() - scanner := bufio.NewScanner(revListReader) - defer func() { - _ = shasToCheckWriter.CloseWithError(scanner.Err()) - }() - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - fields := strings.Split(line, " ") - if len(fields) < 2 || len(fields[1]) == 0 { - continue - } - toWrite := []byte(fields[0] + "\n") - for len(toWrite) > 0 { - n, err := shasToCheckWriter.Write(toWrite) - if err != nil { - _ = revListReader.CloseWithError(err) - break - } - toWrite = toWrite[n:] - } - } -} - -func doCatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { - defer wg.Done() - defer shasToCheckReader.Close() - defer catFileCheckWriter.Close() - - stderr := new(bytes.Buffer) - var errbuf strings.Builder - cmd := git.NewCommand("cat-file", "--batch-check") - if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil { - _ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String())) - } -} - -func doCatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) { - defer wg.Done() - defer catFileCheckWriter.Close() - - stderr := new(bytes.Buffer) - var errbuf strings.Builder - cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") - if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil { - log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String()) - err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String()) - _ = catFileCheckWriter.CloseWithError(err) - errChan <- err - } -} - -func readCatFileBatchCheckAllObjects(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { - defer wg.Done() - defer catFileCheckReader.Close() - scanner := bufio.NewScanner(catFileCheckReader) - defer func() { - _ = shasToBatchWriter.CloseWithError(scanner.Err()) - }() - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - fields := strings.Split(line, " ") - if len(fields) < 3 || fields[1] != "blob" { - continue - } - size, _ := strconv.Atoi(string(fields[2])) - if size > 1024 { - continue - } - toWrite := []byte(fields[0] + "\n") - for len(toWrite) > 0 { - n, err := shasToBatchWriter.Write(toWrite) - if err != nil { - _ = catFileCheckReader.CloseWithError(err) - break - } - toWrite = toWrite[n:] - } - } -} - -func doCatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { - defer wg.Done() - defer shasToBatchReader.Close() - defer catFileBatchWriter.Close() - - stderr := new(bytes.Buffer) - var errbuf strings.Builder - if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil { - _ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())) - } -} - -func readCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { +func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { defer wg.Done() defer catFileBatchReader.Close() contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath} From 4c7dac8c5d28eff5086aad58a87ec38f621fe0bb Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 24 Jul 2019 10:42:08 +0100 Subject: [PATCH 14/19] Add link to find commits on the lfs file view --- templates/repo/settings/lfs_file.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index 2e91559da8b02..6283548eaa9fd 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -7,6 +7,9 @@

    {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}} +
    + {{$.i18n.Tr "repo.settings.lfs_findcommits"}} +

    From 2f55a5f1c4909b30e42c9149439950affa0895f4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 24 Jul 2019 14:01:44 +0100 Subject: [PATCH 15/19] Adjust commit view to state the likely branch causing the commit --- modules/git/pipeline/namerev.go | 28 ++++++++ routers/repo/lfs.go | 78 +++++++++++++++++++--- templates/repo/settings/lfs_file_find.tmpl | 19 ++++-- 3 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 modules/git/pipeline/namerev.go diff --git a/modules/git/pipeline/namerev.go b/modules/git/pipeline/namerev.go new file mode 100644 index 0000000000000..eebb53b0ca209 --- /dev/null +++ b/modules/git/pipeline/namerev.go @@ -0,0 +1,28 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pipeline + +import ( + "bytes" + "fmt" + "io" + "strings" + "sync" + + "code.gitea.io/gitea/modules/git" +) + +// NameRevStdin runs name-rev --stdin +func NameRevStdin(shasToNameReader *io.PipeReader, nameRevStdinWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer shasToNameReader.Close() + defer nameRevStdinWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").RunInDirFullPipeline(tmpBasePath, nameRevStdinWriter, stderr, shasToNameReader); err != nil { + _ = shasToNameReader.CloseWithError(fmt.Errorf("git name-rev [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index e8ae46f03edfe..9a63bafabc440 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -190,14 +190,16 @@ func LFSDelete(ctx *context.Context) { } type lfsResult struct { - Name string - SHA string - Summary string - When time.Time - ParentHashes []plumbing.Hash + Name string + SHA string + Summary string + When time.Time + ParentHashes []plumbing.Hash + BranchName string + FullCommitName string } -type lfsResultSlice []lfsResult +type lfsResultSlice []*lfsResult func (a lfsResultSlice) Len() int { return len(a) } func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -233,8 +235,9 @@ func LFSFileFind(ctx *context.Context) { ctx.Data["SHA"] = sha resultsMap := map[string]*lfsResult{} - results := make([]lfsResult, 0) + results := make([]*lfsResult, 0) + basePath := ctx.Repo.Repository.RepoPath() gogitRepo := ctx.Repo.GitRepo.GoGitRepo() commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ @@ -286,12 +289,71 @@ func LFSFileFind(ctx *context.Context) { } } if !hasParent { - results = append(results, *result) + results = append(results, result) } } sort.Sort(lfsResultSlice(results)) + // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple + shasToNameReader, shasToNameWriter := io.Pipe() + nameRevStdinReader, nameRevStdinWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(3) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(nameRevStdinReader) + i := 0 + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + result := results[i] + result.FullCommitName = line + result.BranchName = strings.Split(line, "~")[0] + i++ + } + }() + go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) + go func() { + defer wg.Done() + defer shasToNameWriter.Close() + for _, result := range results { + i := 0 + if i < len(result.SHA) { + n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) + if err != nil { + errChan <- err + break + } + i += n + } + n := 0 + for n < 1 { + n, err = shasToNameWriter.Write([]byte{'\n'}) + if err != nil { + errChan <- err + break + } + + } + + } + }() + + wg.Wait() + + select { + case err, has := <-errChan: + if has { + ctx.ServerError("LFSPointerFiles", err) + } + default: + } + ctx.Data["Results"] = results ctx.HTML(200, tplSettingsLFSFileFind) } diff --git a/templates/repo/settings/lfs_file_find.tmpl b/templates/repo/settings/lfs_file_find.tmpl index 78ac7739275a9..18db0215a5b7d 100644 --- a/templates/repo/settings/lfs_file_find.tmpl +++ b/templates/repo/settings/lfs_file_find.tmpl @@ -24,17 +24,24 @@
    + {{else}} - + {{end}} From af4a2276ad88e8019159c41f4cd2611260dd4bcb Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 17 Aug 2019 09:42:05 +0100 Subject: [PATCH 16/19] cope with moved UTF8 functions --- routers/repo/lfs.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 9a63bafabc440..555e042802ed3 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -21,13 +21,13 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/pipeline" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/templates" "github.com/Unknwon/com" "github.com/mcuadros/go-version" @@ -120,11 +120,11 @@ func LFSFileGet(ctx *context.Context) { } d, _ := ioutil.ReadAll(dataRc) - buf = templates.ToUTF8WithFallback(append(buf, d...)) + buf = charset.ToUTF8WithFallback(append(buf, d...)) // Building code view blocks with line number on server side. var fileContent string - if content, err := templates.ToUTF8WithErr(buf); err != nil { + if content, err := charset.ToUTF8WithErr(buf); err != nil { log.Error("ToUTF8WithErr: %v", err) fileContent = string(buf) } else { From 56d15f293aa0533a040f26aee3bfed5d653a9e66 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 9 Oct 2019 17:13:44 +0100 Subject: [PATCH 17/19] fix unknwon change --- routers/repo/lfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 555e042802ed3..de5020c944044 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -29,8 +29,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "github.com/Unknwon/com" "github.com/mcuadros/go-version" + "github.com/unknwon/com" gogit "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" From 5a4161d56c9e5e52279b92c6e88a7b1ce802cc34 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 12 Oct 2019 18:56:07 +0100 Subject: [PATCH 18/19] Only read Oid from database --- models/lfs.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/lfs.go b/models/lfs.go index 655fc22dee5e0..5f5fe2ccf430f 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -173,8 +173,10 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { } oids := make([]interface{}, len(metas)) + oidMap := make(map[string]*LFSMetaObject, len(metas)) for i, meta := range metas { oids[i] = meta.Oid + oidMap[meta.Oid] = meta } cond := builder.NewCond() @@ -183,10 +185,11 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user.ID))) } newMetas := make([]*LFSMetaObject, 0, len(metas)) - if err := sess.Cols("oid", "size").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { + if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { return err } for i := range newMetas { + newMetas[i].Size = oidMap[newMetas[i].Oid].Size newMetas[i].RepositoryID = repoID } if _, err := sess.InsertMulti(newMetas); err != nil { From 1856f7752cb3f664c07a51df7bcb1b84e4602618 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 12 Oct 2019 20:44:16 +0100 Subject: [PATCH 19/19] Apply suggestions from code review Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> --- options/locale/locale_en-US.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d6f49046ae711..f4e161750abc7 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1367,8 +1367,8 @@ settings.lfs_no_lfs_files=No LFS files stored in this repository settings.lfs_findcommits=Find commits settings.lfs_lfs_file_no_commits=No Commits found for this LFS file settings.lfs_delete=Delete LFS file with OID %s -settings.lfs_delete_warning=Deleting an LFS file may cause object does not exist errors on checkout. Are you sure? -settings.lfs_findpointerfiles=Find Pointer files +settings.lfs_delete_warning=Deleting an LFS file may cause 'object does not exist' errors on checkout. Are you sure? +settings.lfs_findpointerfiles=Find pointer files settings.lfs_pointers.found=Found %d blob pointer(s) - %d associated, %d unassociated (%d missing from store) settings.lfs_pointers.sha=Blob SHA settings.lfs_pointers.oid=OID
    - - + + {{.Oid}} @@ -51,23 +52,6 @@ {{end}} -
    - -
    - {{template "base/footer" .}} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index 25adaba332faa..88fd08ec91a43 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -6,7 +6,7 @@ {{template "base/alert" .}}

    - {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}} + {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}}

    diff --git a/templates/repo/settings/lfs_pointers.tmpl b/templates/repo/settings/lfs_pointers.tmpl index 6d2f86c0d9431..8a36c6faca522 100644 --- a/templates/repo/settings/lfs_pointers.tmpl +++ b/templates/repo/settings/lfs_pointers.tmpl @@ -44,12 +44,12 @@
    		{{if and .Exists .InRepo}} - - {{.Oid}} + + {{ShortSha .Oid}} {{else}} - - {{.Oid}} + + {{ShortSha .Oid}} {{end}} From efd7f4c3efcd5a8693ea8cba2434a72ea79bf802 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 13 Jun 2019 22:06:26 +0100 Subject: [PATCH 05/19] Add basic repository lfs management --- models/lfs.go | 36 +++++- modules/lfs/server.go | 2 +- modules/repofiles/update.go | 2 +- modules/repofiles/upload.go | 2 +- options/locale/locale_en-US.ini | 4 + routers/repo/lfs.go | 169 ++++++++++++++++++++++++++ routers/routes/routes.go | 7 ++ templates/repo/settings/lfs.tmpl | 85 +++++++++++++ templates/repo/settings/lfs_file.tmpl | 66 ++++++++++ templates/repo/settings/navbar.tmpl | 5 + 10 files changed, 370 insertions(+), 8 deletions(-) create mode 100644 routers/repo/lfs.go create mode 100644 templates/repo/settings/lfs.tmpl create mode 100644 templates/repo/settings/lfs_file.tmpl diff --git a/models/lfs.go b/models/lfs.go index 94d3f579056e7..29f27f09f029d 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -106,21 +106,47 @@ func (repo *Repository) GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error // RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. -func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) error { +func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) (int64, error) { if len(oid) == 0 { - return ErrLFSObjectNotExist + return 0, ErrLFSObjectNotExist } sess := x.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { - return err + return -1, err } m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} if _, err := sess.Delete(m); err != nil { - return err + return -1, err } - return sess.Commit() + count, err := sess.Count(&LFSMetaObject{Oid: oid}) + if err != nil { + return count, err + } + + return count, sess.Commit() +} + +// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository +func (repo *Repository) GetLFSMetaObjects(page, pageSize int) ([]*LFSMetaObject, error) { + sess := x.NewSession() + defer sess.Close() + + if page >= 0 && pageSize > 0 { + start := 0 + if page > 0 { + start = (page - 1) * pageSize + } + sess.Limit(pageSize, start) + } + lfsObjects := make([]*LFSMetaObject, 0, pageSize) + return lfsObjects, sess.Find(&lfsObjects, &LFSMetaObject{RepositoryID: repo.ID}) +} + +// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository +func (repo *Repository) CountLFSMetaObjects() (int64, error) { + return x.Count(&LFSMetaObject{RepositoryID: repo.ID}) } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index bf5355acfc3b1..0bfaa5f35c972 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -330,7 +330,7 @@ func PutHandler(ctx *context.Context) { if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { ctx.Resp.WriteHeader(500) fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) - if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil { + if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil { log.Error("RemoveLFSMetaObjectByOid: %v", err) } return diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index f011017a5e87a..ff4b24adb34b7 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -374,7 +374,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} if !contentStore.Exists(lfsMetaObject) { if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { - if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { + if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) } return nil, err diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 2da101c64dad6..d626ab913ba1f 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -37,7 +37,7 @@ func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, orig continue } if !info.lfsMetaObject.Existing { - if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { + if _, err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { original = fmt.Errorf("%v, %v", original, err) } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 969a0953a2894..3911ca528504b 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1325,6 +1325,10 @@ settings.unarchive.text = Un-Archiving the repo will restore its ability to rece settings.unarchive.success = The repo was successfully un-archived. settings.unarchive.error = An error occurred while trying to un-archive the repo. See the log for more details. settings.update_avatar_success = The repository avatar has been updated. +settings.lfs=LFS +settings.lfs_filelist=LFS files stored in this repository +settings.lfs_delete=Delete LFS file with OID %s +settings.lfs_delete_warning=Deleting an LFS file may cause object does not exist errors on checkout. Are you sure? diff.browse_source = Browse Source diff.parent = parent diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go new file mode 100644 index 0000000000000..f5cf13ce2a3e2 --- /dev/null +++ b/routers/repo/lfs.go @@ -0,0 +1,169 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + "bytes" + "fmt" + gotemplate "html/template" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" +) + +const ( + tplSettingsLFS base.TplName = "repo/settings/lfs" + tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" +) + +// LFSFiles shows a repository's LFS files +func LFSFiles(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFiles", nil) + return + } + page := ctx.QueryInt("page") + if page <= 1 { + page = 1 + } + total, err := ctx.Repo.Repository.CountLFSMetaObjects() + if err != nil { + ctx.ServerError("LFSFiles", err) + return + } + + pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5) + ctx.Data["Title"] = ctx.Tr("repo.settings.lfs") + ctx.Data["PageIsSettingsLFS"] = true + lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum) + if err != nil { + ctx.ServerError("LFSFiles", err) + return + } + ctx.Data["LFSFiles"] = lfsMetaObjects + ctx.Data["Page"] = pager + ctx.HTML(200, tplSettingsLFS) +} + +// LFSFileGet serves a single LFS file +func LFSFileGet(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFileGet", nil) + return + } + ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" + oid := ctx.Params("oid") + ctx.Data["Title"] = oid + ctx.Data["PageIsSettingsLFS"] = true + meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid) + if err != nil { + ctx.ServerError("LFSFileGet", err) + return + } + ctx.Data["LFSFile"] = meta + dataRc, err := lfs.ReadMetaObject(meta) + if err != nil { + ctx.ServerError("LFSFileGet", err) + return + } + defer dataRc.Close() + buf := make([]byte, 1024) + n, err := dataRc.Read(buf) + if err != nil { + ctx.ServerError("Data", err) + return + } + buf = buf[:n] + + isTextFile := base.IsTextFile(buf) + ctx.Data["IsTextFile"] = isTextFile + + fileSize := meta.Size + ctx.Data["FileSize"] = meta.Size + ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct") + switch { + case isTextFile: + if fileSize >= setting.UI.MaxDisplayFileSize { + ctx.Data["IsFileTooLarge"] = true + break + } + + d, _ := ioutil.ReadAll(dataRc) + buf = templates.ToUTF8WithFallback(append(buf, d...)) + + // Building code view blocks with line number on server side. + var fileContent string + if content, err := templates.ToUTF8WithErr(buf); err != nil { + log.Error("ToUTF8WithErr: %v", err) + fileContent = string(buf) + } else { + fileContent = content + } + + var output bytes.Buffer + lines := strings.Split(fileContent, "\n") + //Remove blank line at the end of file + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + for index, line := range lines { + line = gotemplate.HTMLEscapeString(line) + if index != len(lines)-1 { + line += "\n" + } + output.WriteString(fmt.Sprintf(`
  • %s
    • `, index+1, index+1, line)) + } + ctx.Data["FileContent"] = gotemplate.HTML(output.String()) + + output.Reset() + for i := 0; i < len(lines); i++ { + output.WriteString(fmt.Sprintf(`%d`, i+1, i+1)) + } + ctx.Data["LineNums"] = gotemplate.HTML(output.String()) + + case base.IsPDFFile(buf): + ctx.Data["IsPDFFile"] = true + case base.IsVideoFile(buf): + ctx.Data["IsVideoFile"] = true + case base.IsAudioFile(buf): + ctx.Data["IsAudioFile"] = true + case base.IsImageFile(buf): + ctx.Data["IsImageFile"] = true + } + ctx.HTML(200, tplSettingsLFSFile) +} + +// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it +func LFSDelete(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFileGet", nil) + return + } + oid := ctx.Params("oid") + count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid) + if err != nil { + ctx.ServerError("LFSDelete", err) + return + } + // FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here + // Please note a similar condition happens in models/repo.go DeleteRepository + if count == 0 { + oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:]) + err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath)) + if err != nil { + ctx.ServerError("LFSDelete", err) + return + } + } + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 6169aa563c099..03ff9bf8b67fe 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -667,8 +667,15 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/delete", repo.DeleteDeployKey) }) + m.Group("/lfs", func() { + m.Get("", repo.LFSFiles) + m.Get("/show/:oid", repo.LFSFileGet) + m.Post("/delete/:oid", repo.LFSDelete) + }) + }, func(ctx *context.Context) { ctx.Data["PageIsSettings"] = true + ctx.Data["LFSStartServer"] = setting.LFS.StartServer }) }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) diff --git a/templates/repo/settings/lfs.tmpl b/templates/repo/settings/lfs.tmpl new file mode 100644 index 0000000000000..aebf619cd0408 --- /dev/null +++ b/templates/repo/settings/lfs.tmpl @@ -0,0 +1,85 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} + {{template "repo/settings/navbar" .}} +
    + {{template "base/alert" .}} + + + + + + + + {{range .LFSFiles}} + + + + + + + {{end}} + +
    {{.i18n.Tr "repo.settings.lfs_filelist"}}
    + + + {{.Oid}} + + + {{FileSize .Size}}{{TimeSince .CreatedUnix.AsTime $.Lang}} + +
    + {{template "base/paginate" .}} + {{range .LFSFiles}} + + {{end}} +
    + +
    + +
    +
    + +{{template "base/footer" .}} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl new file mode 100644 index 0000000000000..25adaba332faa --- /dev/null +++ b/templates/repo/settings/lfs_file.tmpl @@ -0,0 +1,66 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} + {{template "repo/settings/navbar" .}} +
    + {{template "base/alert" .}} +
    +

    + {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}} +

    +
    +
    + {{if .IsMarkup}} + {{if .FileContent}}{{.FileContent | Safe}}{{end}} + {{else if .IsRenderedHTML}} + 
    {{if .FileContent}}{{.FileContent | Str2html}}{{end}}
    + {{else if not .IsTextFile}} +
    + {{if .IsImageFile}} + + {{else if .IsVideoFile}} + + {{else if .IsAudioFile}} + + {{else if .IsPDFFile}} + + {{else}} + {{.i18n.Tr "repo.file_view_raw"}} + {{end}} +
    + {{else if .FileSize}} + + + + {{if .IsFileTooLarge}} + + {{else}} + + + {{end}} + + +
    {{.i18n.Tr "repo.file_too_large"}}{{.LineNums}}
      {{.FileContent}}
    + {{end}} +
    +
    +
    +
    +
    + +{{template "base/footer" .}} diff --git a/templates/repo/settings/navbar.tmpl b/templates/repo/settings/navbar.tmpl index 24082000e2ae9..abd6e285dc3e3 100644 --- a/templates/repo/settings/navbar.tmpl +++ b/templates/repo/settings/navbar.tmpl @@ -21,4 +21,9 @@ {{.i18n.Tr "repo.settings.deploy_keys"}} + {{if .LFSStartServer}} + + {{.i18n.Tr "repo.settings.lfs"}} + + {{end}} From fbd8b189bf11b5b457beb9530d610c22355a2cc6 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 14 Jun 2019 17:18:12 +0100 Subject: [PATCH 06/19] add auto-associate function --- models/lfs.go | 44 ++++ models/repo_list.go | 48 ++-- options/locale/locale_en-US.ini | 8 + public/css/index.css | 1 + public/less/_base.less | 10 + routers/repo/lfs.go | 253 +++++++++++++++++++++- routers/routes/routes.go | 2 + templates/repo/settings/lfs.tmpl | 34 +-- templates/repo/settings/lfs_pointers.tmpl | 67 ++++++ 9 files changed, 418 insertions(+), 49 deletions(-) create mode 100644 templates/repo/settings/lfs_pointers.tmpl diff --git a/models/lfs.go b/models/lfs.go index 29f27f09f029d..ee9132265abe4 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -8,6 +8,7 @@ import ( "io" "code.gitea.io/gitea/modules/util" + "github.com/go-xorm/builder" ) // LFSMetaObject stores metadata for LFS tracked files. @@ -150,3 +151,46 @@ func (repo *Repository) GetLFSMetaObjects(page, pageSize int) ([]*LFSMetaObject, func (repo *Repository) CountLFSMetaObjects() (int64, error) { return x.Count(&LFSMetaObject{RepositoryID: repo.ID}) } + +// LFSObjectAccessible checks if a provided Oid is accessible to the user +func LFSObjectAccessible(user *User, oid string) (bool, error) { + if user.IsAdmin { + count, err := x.Count(&LFSMetaObject{Oid: oid}) + return (count > 0), err + } + cond := accessibleRepositoryCondition(user.ID) + count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid}) + return (count > 0), err +} + +// LFSAutoAssociate auto associates accessible LFSMetaObjects +func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + oids := make([]interface{}, len(metas)) + for i, meta := range metas { + oids[i] = meta.Oid + } + + cond := builder.NewCond() + if !user.IsAdmin { + cond = builder.In("`lfs_meta_object`.repository_id", + builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user.ID))) + } + newMetas := make([]*LFSMetaObject, 0, len(metas)) + if err := sess.Cols("oid", "size").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { + return err + } + for i := range newMetas { + newMetas[i].RepositoryID = repoID + } + if _, err := sess.InsertMulti(newMetas); err != nil { + return err + } + + return sess.Commit() +} diff --git a/models/repo_list.go b/models/repo_list.go index 7460c4b0ede61..73b79a1fdc56a 100644 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -174,28 +174,7 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err if opts.Private { if !opts.UserIsAdmin && opts.UserID != 0 && opts.UserID != opts.OwnerID { // OK we're in the context of a User - // We should be Either - cond = cond.And(builder.Or( - // 1. Be able to see all non-private repositories that either: - cond.And( - builder.Eq{"is_private": false}, - builder.Or( - // A. Aren't in organisations __OR__ - builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})), - // B. Isn't a private organisation. (Limited is OK because we're logged in) - builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))), - ), - // 2. Be able to see all repositories that we have access to - builder.In("id", builder.Select("repo_id"). - From("`access`"). - Where(builder.And( - builder.Eq{"user_id": opts.UserID}, - builder.Gt{"mode": int(AccessModeNone)}))), - // 3. Be able to see all repositories that we are in a team - builder.In("id", builder.Select("`team_repo`.repo_id"). - From("team_repo"). - Where(builder.Eq{"`team_user`.uid": opts.UserID}). - Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id")))) + cond = cond.And(accessibleRepositoryCondition(opts.UserID)) } } else { // Not looking at private organisations @@ -307,6 +286,31 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err return repos, count, nil } +// accessibleRepositoryCondition takes a user a returns a condition for checking if a repository is accessible +func accessibleRepositoryCondition(userID int64) builder.Cond { + return builder.Or( + // 1. Be able to see all non-private repositories that either: + builder.And( + builder.Eq{"`repository`.is_private": false}, + builder.Or( + // A. Aren't in organisations __OR__ + builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})), + // B. Isn't a private organisation. (Limited is OK because we're logged in) + builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))), + ), + // 2. Be able to see all repositories that we have access to + builder.In("`repository`.id", builder.Select("repo_id"). + From("`access`"). + Where(builder.And( + builder.Eq{"user_id": userID}, + builder.Gt{"mode": int(AccessModeNone)}))), + // 3. Be able to see all repositories that we are in a team + builder.In("`repository`.id", builder.Select("`team_repo`.repo_id"). + From("team_repo"). + Where(builder.Eq{"`team_user`.uid": userID}). + Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id"))) +} + // FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) { var accessCond builder.Cond = builder.Eq{"is_private": false} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3911ca528504b..2deea76a47cbb 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1329,6 +1329,14 @@ settings.lfs=LFS settings.lfs_filelist=LFS files stored in this repository settings.lfs_delete=Delete LFS file with OID %s settings.lfs_delete_warning=Deleting an LFS file may cause object does not exist errors on checkout. Are you sure? +settings.lfs_findpointerfiles=Find Pointer files +settings.lfs_pointers.found=Found %d blob pointer(s) - %d associated, %d unassociated (%d missing from store) +settings.lfs_pointers.sha=SHA +settings.lfs_pointers.oid=OID +settings.lfs_pointers.inRepo=In Repo +settings.lfs_pointers.exists=Exists in store +settings.lfs_pointers.accessible=Accessible to User +settings.lfs_pointers.associateAccessible=Associate accessible %d OIDs diff.browse_source = Browse Source diff.parent = parent diff --git a/public/css/index.css b/public/css/index.css index 96dfcb0e8b7e3..7347884162080 100644 --- a/public/css/index.css +++ b/public/css/index.css @@ -126,6 +126,7 @@ a{cursor:pointer} .ui .form .fake{display:none!important} .ui .form .sub.field{margin-left:25px} .ui .sha.label{font-family:'SF Mono',Consolas,Menlo,'Liberation Mono',Monaco,'Lucida Console',monospace;font-size:13px;padding:6px 10px 4px 10px;font-weight:400;margin:0 6px} +.ui .button.truncate{display:inline-block;max-width:100%;overflow:hidden;text-overflow:ellipsis;vertical-align:top;white-space:nowrap;margin-right:6px} .ui.status.buttons .octicon{margin-right:4px} .ui.inline.delete-button{padding:8px 15px;font-weight:400} .ui .background.red{background-color:#d95c5c!important} diff --git a/public/less/_base.less b/public/less/_base.less index ddf174cf475f3..f539855aa2f59 100644 --- a/public/less/_base.less +++ b/public/less/_base.less @@ -539,6 +539,16 @@ code, margin: 0 6px; } + .button.truncate { + display: inline-block; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + vertical-align: top; + white-space: nowrap; + margin-right: 6px; + } + &.status.buttons { .octicon { margin-right: 4px; diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index f5cf13ce2a3e2..05cc0aa7c4a53 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -5,25 +5,34 @@ package repo import ( + "bufio" "bytes" "fmt" gotemplate "html/template" + "io" "io/ioutil" "os" "path/filepath" + "strconv" "strings" + "sync" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "github.com/Unknwon/com" + "github.com/mcuadros/go-version" ) const ( - tplSettingsLFS base.TplName = "repo/settings/lfs" - tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" + tplSettingsLFS base.TplName = "repo/settings/lfs" + tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" + tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers" ) // LFSFiles shows a repository's LFS files @@ -67,6 +76,10 @@ func LFSFileGet(ctx *context.Context) { ctx.Data["PageIsSettingsLFS"] = true meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid) if err != nil { + if err == models.ErrLFSObjectNotExist { + ctx.NotFound("LFSFileGet", nil) + return + } ctx.ServerError("LFSFileGet", err) return } @@ -167,3 +180,239 @@ func LFSDelete(ctx *context.Context) { } ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") } + +// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store +func LFSPointerFiles(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFileGet", nil) + return + } + ctx.Data["PageIsSettingsLFS"] = true + binVersion, err := git.BinVersion() + if err != nil { + log.Fatal("Error retrieving git version: %v", err) + } + + if !version.Compare(binVersion, "2.6.0", ">=") { + ctx.ServerError("LFSPointerFiles", fmt.Errorf("Git version too low")) + return + } + + basePath := ctx.Repo.Repository.RepoPath() + + pointerChan := make(chan pointerResult) + + catFileCheckReader, catFileCheckWriter := io.Pipe() + shasToBatchReader, shasToBatchWriter := io.Pipe() + catFileBatchReader, catFileBatchWriter := io.Pipe() + wg := sync.WaitGroup{} + wg.Add(5) + + var numPointers, numAssociated, numNoExist, numAssociatable int + + go func() { + defer wg.Done() + pointers := make([]pointerResult, 0, 50) + for pointer := range pointerChan { + pointers = append(pointers, pointer) + if pointer.InRepo { + numAssociated++ + } + if !pointer.Exists { + numNoExist++ + } + if !pointer.InRepo && pointer.Accessible { + numAssociatable++ + } + } + numPointers = len(pointers) + ctx.Data["Pointers"] = pointers + ctx.Data["NumPointers"] = numPointers + ctx.Data["NumAssociated"] = numAssociated + ctx.Data["NumAssociatable"] = numAssociatable + ctx.Data["NumNoExist"] = numNoExist + ctx.Data["NumNotAssociated"] = numPointers - numAssociated + }() + go readCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) + go doCatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) + go readCatFileBatchCheckAllObjects(catFileCheckReader, shasToBatchWriter, &wg) + go doCatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath) + + wg.Wait() + ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" + ctx.HTML(200, tplSettingsLFSPointers) +} + +type pointerResult struct { + SHA string + Oid string + Size int64 + InRepo bool + Exists bool + Accessible bool +} + +func doCatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer catFileCheckWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") + if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil { + _ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} + +func readCatFileBatchCheckAllObjects(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { + defer wg.Done() + defer catFileCheckReader.Close() + scanner := bufio.NewScanner(catFileCheckReader) + defer func() { + _ = shasToBatchWriter.CloseWithError(scanner.Err()) + }() + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 3 || fields[1] != "blob" { + continue + } + size, _ := strconv.Atoi(string(fields[2])) + if size > 1024 { + continue + } + toWrite := []byte(fields[0] + "\n") + for len(toWrite) > 0 { + n, err := shasToBatchWriter.Write(toWrite) + if err != nil { + _ = catFileCheckReader.CloseWithError(err) + break + } + toWrite = toWrite[n:] + } + } +} + +func doCatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { + defer wg.Done() + defer shasToBatchReader.Close() + defer catFileBatchWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil { + _ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())) + } +} + +func readCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { + defer wg.Done() + defer catFileBatchReader.Close() + contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath} + + bufferedReader := bufio.NewReader(catFileBatchReader) + buf := make([]byte, 1025) + for { + // File descriptor line: sha + sha, err := bufferedReader.ReadString(' ') + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + // Throw away the blob + if _, err := bufferedReader.ReadString(' '); err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + sizeStr, err := bufferedReader.ReadString('\n') + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + pointerBuf := buf[:size+1] + if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + pointerBuf = pointerBuf[:size] + // Now we need to check if the pointerBuf is an LFS pointer + pointer := lfs.IsPointerFile(&pointerBuf) + if pointer == nil { + continue + } + + result := pointerResult{ + SHA: strings.TrimSpace(sha), + Oid: pointer.Oid, + Size: pointer.Size, + } + + // Then we need to check that this pointer is in the db + if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { + if err != models.ErrLFSObjectNotExist { + _ = catFileBatchReader.CloseWithError(err) + break + } + } else { + result.InRepo = true + } + + result.Exists = contentStore.Exists(pointer) + + if result.Exists { + if !result.InRepo { + // Can we fix? + // OK well that's "simple" + // - we need to check whether current user has access to a repo that has access to the file + result.Accessible, err = models.LFSObjectAccessible(user, result.Oid) + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + } else { + result.Accessible = true + } + } + pointerChan <- result + } + close(pointerChan) +} + +// LFSAutoAssociate auto associates accessible lfs files +func LFSAutoAssociate(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSAutoAssociate", nil) + return + } + oids := ctx.QueryStrings("oid") + metas := make([]*models.LFSMetaObject, len(oids)) + for i, oid := range oids { + idx := strings.IndexRune(oid, ' ') + if idx < 0 || idx+1 > len(oid) { + ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid)) + return + } + var err error + metas[i] = &models.LFSMetaObject{} + metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64() + if err != nil { + ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err)) + return + } + metas[i].Oid = oid[:idx] + //metas[i].RepositoryID = ctx.Repo.Repository.ID + } + if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil { + ctx.ServerError("LFSAutoAssociate", err) + return + } + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 03ff9bf8b67fe..28edc2d5a5e94 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -671,6 +671,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", repo.LFSFiles) m.Get("/show/:oid", repo.LFSFileGet) m.Post("/delete/:oid", repo.LFSDelete) + m.Get("/pointers", repo.LFSPointerFiles) + m.Post("/pointers/associate", repo.LFSAutoAssociate) }) }, func(ctx *context.Context) { diff --git a/templates/repo/settings/lfs.tmpl b/templates/repo/settings/lfs.tmpl index aebf619cd0408..630b446027001 100644 --- a/templates/repo/settings/lfs.tmpl +++ b/templates/repo/settings/lfs.tmpl @@ -4,18 +4,19 @@ {{template "repo/settings/navbar" .}}
    {{template "base/alert" .}} - - - - - - +

    + {{.i18n.Tr "repo.settings.lfs_filelist"}} +
    + {{.i18n.Tr "repo.settings.lfs_findpointerfiles"}} +
    +

    +
    {{.i18n.Tr "repo.settings.lfs_filelist"}}
    {{range .LFSFiles}} - + {{else}} + + + {{end}}
    - - + + {{.Oid}} @@ -51,23 +52,6 @@ {{end}} -
    - -
    - {{template "base/footer" .}} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index 25adaba332faa..88fd08ec91a43 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -6,7 +6,7 @@ {{template "base/alert" .}}

    - {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}} + {{.i18n.Tr "repo.settings.lfs"}} / {{.LFSFile.Oid}}

    diff --git a/templates/repo/settings/lfs_pointers.tmpl b/templates/repo/settings/lfs_pointers.tmpl index 6d2f86c0d9431..8a36c6faca522 100644 --- a/templates/repo/settings/lfs_pointers.tmpl +++ b/templates/repo/settings/lfs_pointers.tmpl @@ -44,12 +44,12 @@
    		{{if and .Exists .InRepo}} - - {{.Oid}} + + {{ShortSha .Oid}} {{else}} - - {{.Oid}} + + {{ShortSha .Oid}} {{end}} From fa5be8c5d0a529303d156be0279ca327c60d3eb4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 22 Jul 2019 15:09:18 +0100 Subject: [PATCH 09/19] update to use xorm.io/builder --- models/lfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/lfs.go b/models/lfs.go index ee9132265abe4..e5b2a717f7054 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -8,7 +8,7 @@ import ( "io" "code.gitea.io/gitea/modules/util" - "github.com/go-xorm/builder" + "xorm.io/builder" ) // LFSMetaObject stores metadata for LFS tracked files. From 280aa5ea38e4bd0dbd0be02cc0db3688553fac7e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 22 Jul 2019 22:37:52 +0100 Subject: [PATCH 10/19] Add functionality to find commits with this lfs file --- modules/git/repo.go | 5 ++ options/locale/locale_en-US.ini | 5 +- routers/repo/lfs.go | 86 +++++++++++++++++++++- routers/routes/routes.go | 1 + templates/repo/settings/lfs.tmpl | 7 +- templates/repo/settings/lfs_file.tmpl | 12 --- templates/repo/settings/lfs_file_find.tmpl | 45 +++++++++++ templates/repo/settings/lfs_pointers.tmpl | 6 +- 8 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 templates/repo/settings/lfs_file_find.tmpl diff --git a/modules/git/repo.go b/modules/git/repo.go index 8a40fb1b91ba2..a792f31049195 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -107,6 +107,11 @@ func OpenRepository(repoPath string) (*Repository, error) { }, nil } +// GoGitRepo gets the go-git repo representation +func (repo *Repository) GoGitRepo() *gogit.Repository { + return repo.gogitRepo +} + // IsEmpty Check if repository is empty. func (repo *Repository) IsEmpty() (bool, error) { var errbuf strings.Builder diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 2deea76a47cbb..687d244c21bf1 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1327,11 +1327,14 @@ settings.unarchive.error = An error occurred while trying to un-archive the repo settings.update_avatar_success = The repository avatar has been updated. settings.lfs=LFS settings.lfs_filelist=LFS files stored in this repository +settings.lfs_no_lfs_files=No LFS files stored in this repository +settings.lfs_findcommits=Find commits +settings.lfs_lfs_file_no_commits=No Commits found for this LFS file settings.lfs_delete=Delete LFS file with OID %s settings.lfs_delete_warning=Deleting an LFS file may cause object does not exist errors on checkout. Are you sure? settings.lfs_findpointerfiles=Find Pointer files settings.lfs_pointers.found=Found %d blob pointer(s) - %d associated, %d unassociated (%d missing from store) -settings.lfs_pointers.sha=SHA +settings.lfs_pointers.sha=Blob SHA settings.lfs_pointers.oid=OID settings.lfs_pointers.inRepo=In Repo settings.lfs_pointers.exists=Exists in store diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 7d6dd15d94836..ad1015fb123e5 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" "sync" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" @@ -25,13 +26,18 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "github.com/Unknwon/com" "github.com/mcuadros/go-version" + gogit "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) const ( tplSettingsLFS base.TplName = "repo/settings/lfs" tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" + tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find" tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers" ) @@ -159,7 +165,7 @@ func LFSFileGet(ctx *context.Context) { // LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it func LFSDelete(ctx *context.Context) { if !setting.LFS.StartServer { - ctx.NotFound("LFSFileGet", nil) + ctx.NotFound("LFSDelete", nil) return } oid := ctx.Params("oid") @@ -181,6 +187,84 @@ func LFSDelete(ctx *context.Context) { ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") } +type lfsResult struct { + Name string + SHA string + Summary string + When time.Time +} + +// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha +func LFSFileFind(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSFind", nil) + return + } + oid := ctx.Query("oid") + size := ctx.QueryInt64("size") + if len(oid) == 0 || size == 0 { + ctx.NotFound("LFSFind", nil) + return + } + sha := ctx.Query("sha") + ctx.Data["Title"] = oid + ctx.Data["PageIsSettingsLFS"] = true + var hash plumbing.Hash + if len(sha) == 0 { + meta := models.LFSMetaObject{Oid: oid, Size: size} + pointer := meta.Pointer() + hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer)) + sha = hash.String() + } else { + hash = plumbing.NewHash(sha) + } + ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" + ctx.Data["Oid"] = oid + ctx.Data["Size"] = size + ctx.Data["SHA"] = sha + + results := make([]lfsResult, 0) + + gogitRepo := ctx.Repo.GitRepo.GoGitRepo() + + commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ + Order: gogit.LogOrderCommitterTime, + All: true, + }) + if err != nil { + log.Error("Failed to get GoGit CommitsIter: %v", err) + ctx.ServerError("LFSFind: Iterate Commits", err) + return + } + + commitsIter.ForEach(func(gitCommit *object.Commit) error { + tree, err := gitCommit.Tree() + if err != nil { + return err + } + treeWalker := object.NewTreeWalker(tree, true, nil) + defer treeWalker.Close() + for { + name, entry, err := treeWalker.Next() + if err == io.EOF { + break + } + if entry.Hash == hash { + results = append(results, lfsResult{ + Name: name, + SHA: gitCommit.Hash.String(), + Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], + When: gitCommit.Author.When, + }) + } + } + return nil + }) + + ctx.Data["Results"] = results + ctx.HTML(200, tplSettingsLFSFileFind) +} + // LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store func LFSPointerFiles(ctx *context.Context) { if !setting.LFS.StartServer { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 28edc2d5a5e94..b84d085744f5e 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -673,6 +673,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/delete/:oid", repo.LFSDelete) m.Get("/pointers", repo.LFSPointerFiles) m.Post("/pointers/associate", repo.LFSAutoAssociate) + m.Get("/find", repo.LFSFileFind) }) }, func(ctx *context.Context) { diff --git a/templates/repo/settings/lfs.tmpl b/templates/repo/settings/lfs.tmpl index c1fce698f2e5b..e4480a8b97220 100644 --- a/templates/repo/settings/lfs.tmpl +++ b/templates/repo/settings/lfs.tmpl @@ -23,12 +23,17 @@ {{FileSize .Size}} {{TimeSince .CreatedUnix.AsTime $.Lang}} + + {{$.i18n.Tr "repo.settings.lfs_findcommits"}}
    {{.i18n.Tr "repo.settings.lfs_no_lfs_files"}}
    diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index 88fd08ec91a43..2e91559da8b02 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -51,16 +51,4 @@
    - {{template "base/footer" .}} diff --git a/templates/repo/settings/lfs_file_find.tmpl b/templates/repo/settings/lfs_file_find.tmpl new file mode 100644 index 0000000000000..78ac7739275a9 --- /dev/null +++ b/templates/repo/settings/lfs_file_find.tmpl @@ -0,0 +1,45 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} + {{template "repo/settings/navbar" .}} +
    + {{template "base/alert" .}} +
    +

    + {{.i18n.Tr "repo.settings.lfs"}} / {{.Oid}} +

    + + + {{range .Results}} + + + + + + + {{else}} + + + + {{end}} + +
    + + {{.Name}} + + + + {{.Summary}} + + + + + + {{ShortSha .SHA}} + + + {{TimeSince .When $.Lang}}
    {{.i18n.Tr "repo.settings.lfs_lfs_file_no_commits"}}
    +
    +
    +
    +{{template "base/footer" .}} diff --git a/templates/repo/settings/lfs_pointers.tmpl b/templates/repo/settings/lfs_pointers.tmpl index 8a36c6faca522..d8c0baadec2b9 100644 --- a/templates/repo/settings/lfs_pointers.tmpl +++ b/templates/repo/settings/lfs_pointers.tmpl @@ -25,7 +25,8 @@
    {{.i18n.Tr "repo.settings.lfs_pointers.sha"}}{{.i18n.Tr "repo.settings.lfs_pointers.oid"}}{{.i18n.Tr "repo.settings.lfs_pointers.oid"}} {{.i18n.Tr "repo.settings.lfs_pointers.inRepo"}} {{.i18n.Tr "repo.settings.lfs_pointers.exists"}} {{.i18n.Tr "repo.settings.lfs_pointers.accessible"}} + {{$.i18n.Tr "repo.settings.lfs_findcommits"}} +
    {{.i18n.Tr "repo.settings.lfs_pointers.sha"}}{{.i18n.Tr "repo.settings.lfs_pointers.oid"}}{{.i18n.Tr "repo.settings.lfs_pointers.oid"}} {{.i18n.Tr "repo.settings.lfs_pointers.inRepo"}} {{.i18n.Tr "repo.settings.lfs_pointers.exists"}} {{.i18n.Tr "repo.settings.lfs_pointers.accessible"}} - - - {{ShortSha .SHA}} - - + {{.BranchName}} + + {{if .ParentHashes}} + {{$.i18n.Tr "repo.diff.parent"}} + {{range .ParentHashes}} + {{ShortSha .String}} + {{end}} + {{end}} +
    + {{$.i18n.Tr "repo.diff.commit"}} + {{ShortSha .SHA}}     		{{TimeSince .When $.Lang}}
    {{.i18n.Tr "repo.settings.lfs_lfs_file_no_commits"}}{{.i18n.Tr "repo.settings.lfs_lfs_file_no_commits"}}