Skip to content

Cache GPG keys, emails and users when list commits #34086

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Apr 9, 2025
Merged
27 changes: 13 additions & 14 deletions models/user/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -1187,29 +1187,28 @@ func GetUsersByEmails(ctx context.Context, emails []string) (map[string]*User, e
for _, email := range emailAddresses {
userIDs.Add(email.UID)
}
users, err := GetUsersMapByIDs(ctx, userIDs.Values())
if err != nil {
return nil, err
}

results := make(map[string]*User, len(emails))
for _, email := range emailAddresses {
user := users[email.UID]
if user != nil {
if user.KeepEmailPrivate {
results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user
} else {
results[email.Email] = user

if len(userIDs) > 0 {
users, err := GetUsersMapByIDs(ctx, userIDs.Values())
if err != nil {
return nil, err
}

for _, email := range emailAddresses {
user := users[email.UID]
if user != nil {
results[user.GetEmail()] = user
}
}
}

users = make(map[int64]*User, len(needCheckUserNames))
users := make(map[int64]*User, len(needCheckUserNames))
if err := db.GetEngine(ctx).In("lower_name", needCheckUserNames.Values()).Find(&users); err != nil {
return nil, err
}
for _, user := range users {
results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user
results[user.GetPlaceholderEmail()] = user
}
return results, nil
}
Expand Down
2 changes: 2 additions & 0 deletions routers/private/hook_verification.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"io"
"os"

"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
asymkey_service "code.gitea.io/gitea/services/asymkey"
Expand Down Expand Up @@ -95,6 +96,7 @@ func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error {
if err != nil {
return err
}
ctx = cache.WithCacheContext(ctx)
verification := asymkey_service.ParseCommitWithSignature(ctx, commit)
if !verification.Verified {
cancel()
Expand Down
3 changes: 2 additions & 1 deletion routers/web/repo/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
unit_model "code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
Expand Down Expand Up @@ -384,7 +385,7 @@ func Diff(ctx *context.Context) {
ctx.Data["CommitStatus"] = git_model.CalcCommitStatus(statuses)
ctx.Data["CommitStatuses"] = statuses

verification := asymkey_service.ParseCommitWithSignature(ctx, commit)
verification := asymkey_service.ParseCommitWithSignature(cache.WithCacheContext(ctx), commit)
ctx.Data["Verification"] = verification
ctx.Data["Author"] = user_model.ValidateCommitWithEmail(ctx, commit)
ctx.Data["Parents"] = parents
Expand Down
3 changes: 2 additions & 1 deletion routers/web/repo/view.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
unit_model "code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/fileicon"
"code.gitea.io/gitea/modules/git"
Expand Down Expand Up @@ -120,7 +121,7 @@ func loadLatestCommitData(ctx *context.Context, latestCommit *git.Commit) bool {
// or of directory if not in root directory.
ctx.Data["LatestCommit"] = latestCommit
if latestCommit != nil {
verification := asymkey_service.ParseCommitWithSignature(ctx, latestCommit)
verification := asymkey_service.ParseCommitWithSignature(cache.WithCacheContext(ctx), latestCommit)

if err := asymkey_model.CalculateTrustStatus(verification, ctx.Repo.Repository.GetTrustModel(), func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, ctx.Repo.Repository, user.ID)
Expand Down
46 changes: 33 additions & 13 deletions services/asymkey/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Expand Down Expand Up @@ -47,6 +48,12 @@ func ParseCommitWithSignature(ctx context.Context, c *git.Commit) *asymkey_model
return ParseCommitWithSignatureCommitter(ctx, c, committer)
}

const (
cacheUserEmailAddressKey = "gpg_user_email_address"
cacheUserKey = "gpg_user"
cacheGPGListKey = "gpg_key_list"
)

func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, committer *user_model.User) *asymkey_model.CommitVerification {
// If no signature just report the committer
if c.Signature == nil {
Expand Down Expand Up @@ -115,7 +122,9 @@ func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, commi
}
}

committerEmailAddresses, _ := user_model.GetEmailAddresses(ctx, committer.ID)
committerEmailAddresses, _ := cache.GetWithContextCache(ctx, cacheUserEmailAddressKey, committer.ID, func() ([]*user_model.EmailAddress, error) {
return user_model.GetEmailAddresses(ctx, committer.ID)
})
activated := false
for _, e := range committerEmailAddresses {
if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) {
Expand Down Expand Up @@ -209,10 +218,13 @@ func checkKeyEmails(ctx context.Context, email string, keys ...*asymkey_model.GP
}
if key.Verified && key.OwnerID != 0 {
if uid != key.OwnerID {
userEmails, _ = user_model.GetEmailAddresses(ctx, key.OwnerID)
userEmails, _ = cache.GetWithContextCache(ctx, cacheUserEmailAddressKey, key.OwnerID, func() ([]*user_model.EmailAddress, error) {
return user_model.GetEmailAddresses(ctx, key.OwnerID)
})
uid = key.OwnerID
user = &user_model.User{ID: uid}
_, _ = user_model.GetUser(ctx, user)
user, _ = cache.GetWithContextCache(ctx, cacheUserKey, uid, func() (*user_model.User, error) {
return user_model.GetUserByID(ctx, uid)
})
}
for _, e := range userEmails {
if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) {
Expand All @@ -231,9 +243,11 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
if keyID == "" {
return nil
}
keys, err := db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: keyID,
IncludeSubKeys: true,
keys, err := cache.GetWithContextCache(ctx, cacheGPGListKey, keyID, func() ([]*asymkey_model.GPGKey, error) {
return db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: keyID,
IncludeSubKeys: true,
})
})
if err != nil {
log.Error("GetGPGKeysByKeyID: %v", err)
Expand All @@ -249,9 +263,11 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
for _, key := range keys {
var primaryKeys []*asymkey_model.GPGKey
if key.PrimaryKeyID != "" {
primaryKeys, err = db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: key.PrimaryKeyID,
IncludeSubKeys: true,
primaryKeys, err = cache.GetWithContextCache(ctx, cacheGPGListKey, key.PrimaryKeyID, func() ([]*asymkey_model.GPGKey, error) {
return db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: key.PrimaryKeyID,
IncludeSubKeys: true,
})
})
if err != nil {
log.Error("GetGPGKeysByKeyID: %v", err)
Expand All @@ -272,8 +288,10 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
Name: name,
Email: email,
}
if key.OwnerID != 0 {
owner, err := user_model.GetUserByID(ctx, key.OwnerID)
if key.OwnerID > 0 {
owner, err := cache.GetWithContextCache(ctx, cacheUserKey, key.OwnerID, func() (*user_model.User, error) {
return user_model.GetUserByID(ctx, key.OwnerID)
})
if err == nil {
signer = owner
} else if !user_model.IsErrUserNotExist(err) {
Expand Down Expand Up @@ -381,7 +399,9 @@ func ParseCommitWithSSHSignature(ctx context.Context, c *git.Commit, committer *
}
}

committerEmailAddresses, err := user_model.GetEmailAddresses(ctx, committer.ID)
committerEmailAddresses, err := cache.GetWithContextCache(ctx, cacheUserEmailAddressKey, committer.ID, func() ([]*user_model.EmailAddress, error) {
return user_model.GetEmailAddresses(ctx, committer.ID)
})
if err != nil {
log.Error("GetEmailAddresses: %v", err)
}
Expand Down
5 changes: 5 additions & 0 deletions services/asymkey/sign.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
issues_model "code.gitea.io/gitea/models/issues"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
Expand Down Expand Up @@ -177,6 +178,8 @@ func SignWikiCommit(ctx context.Context, repo *repo_model.Repository, u *user_mo
return false, "", nil, &ErrWontSign{noKey}
}

ctx = cache.WithCacheContext(ctx)

Loop:
for _, rule := range rules {
switch rule {
Expand Down Expand Up @@ -232,6 +235,7 @@ func SignCRUDAction(ctx context.Context, repoPath string, u *user_model.User, tm
if signingKey == "" {
return false, "", nil, &ErrWontSign{noKey}
}
ctx = cache.WithCacheContext(ctx)

Loop:
for _, rule := range rules {
Expand Down Expand Up @@ -297,6 +301,7 @@ func SignMerge(ctx context.Context, pr *issues_model.PullRequest, u *user_model.

var gitRepo *git.Repository
var err error
ctx = cache.WithCacheContext(ctx)

Loop:
for _, rule := range rules {
Expand Down
3 changes: 2 additions & 1 deletion services/convert/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
Expand Down Expand Up @@ -254,7 +255,7 @@ func ToActionArtifact(repo *repo_model.Repository, art *actions_model.ActionArti

// ToVerification convert a git.Commit.Signature to an api.PayloadCommitVerification
func ToVerification(ctx context.Context, c *git.Commit) *api.PayloadCommitVerification {
verif := asymkey_service.ParseCommitWithSignature(ctx, c)
verif := asymkey_service.ParseCommitWithSignature(cache.WithCacheContext(ctx), c)
commitVerification := &api.PayloadCommitVerification{
Verified: verif.Verified,
Reason: verif.Reason,
Expand Down
13 changes: 9 additions & 4 deletions services/git/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ import (
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
asymkey_service "code.gitea.io/gitea/services/asymkey"
)

// ParseCommitsWithSignature checks if signaute of commits are corresponding to users gpg keys.
func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType, isOwnerMemberCollaborator func(*user_model.User) (bool, error)) ([]*asymkey_model.SignCommit, error) {
func ParseCommitsWithSignature(ctx context.Context, repo *repo_model.Repository, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType) ([]*asymkey_model.SignCommit, error) {
newCommits := make([]*asymkey_model.SignCommit, 0, len(oldCommits))
keyMap := map[string]bool{}

Expand All @@ -33,6 +34,8 @@ func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.Use
return nil, err
}

ctx = cache.WithCacheContext(ctx)

for _, c := range oldCommits {
committer, ok := emailUsers[c.Committer.Email]
if !ok && c.Committer != nil {
Expand All @@ -47,6 +50,10 @@ func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.Use
Verification: asymkey_service.ParseCommitWithSignatureCommitter(ctx, c.Commit, committer),
}

isOwnerMemberCollaborator := func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
}

_ = asymkey_model.CalculateTrustStatus(signCommit.Verification, repoTrustModel, isOwnerMemberCollaborator, &keyMap)

newCommits = append(newCommits, signCommit)
Expand All @@ -62,11 +69,9 @@ func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo
}
signedCommits, err := ParseCommitsWithSignature(
ctx,
repo,
validatedCommits,
repo.GetTrustModel(),
func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
},
)
if err != nil {
return nil, err
Expand Down
3 changes: 2 additions & 1 deletion services/repository/files/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"

repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/structs"
asymkey_service "code.gitea.io/gitea/services/asymkey"
Expand All @@ -24,7 +25,7 @@ func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, bra
// GetPayloadCommitVerification returns the verification information of a commit
func GetPayloadCommitVerification(ctx context.Context, commit *git.Commit) *structs.PayloadCommitVerification {
verification := &structs.PayloadCommitVerification{}
commitVerification := asymkey_service.ParseCommitWithSignature(ctx, commit)
commitVerification := asymkey_service.ParseCommitWithSignature(cache.WithCacheContext(ctx), commit)
if commit.Signature != nil {
verification.Signature = commit.Signature.Signature
verification.Payload = commit.Signature.Payload
Expand Down
2 changes: 2 additions & 0 deletions services/repository/gitgraph/graph_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
asymkey_service "code.gitea.io/gitea/services/asymkey"
Expand Down Expand Up @@ -97,6 +98,7 @@ func (graph *Graph) LoadAndProcessCommits(ctx context.Context, repository *repo_

emails := map[string]*user_model.User{}
keyMap := map[string]bool{}
ctx = cache.WithCacheContext(ctx)

for _, c := range graph.Commits {
if len(c.Rev) == 0 {
Expand Down