Support repo license

build/generate-licenses.go

@@ -5,6 +5,8 @@ package main
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/md5"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -15,19 +17,22 @@ import (
 	"path/filepath"
 	"strings"
 
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/util"
 )
 
 func main() {
 	var (
-		prefix         = "gitea-licenses"
-		url            = "https://api.github.com/repos/spdx/license-list-data/tarball"
-		githubApiToken = ""
-		githubUsername = ""
-		destination    = ""
+		prefix                    = "gitea-licenses"
+		url                       = "https://api.github.com/repos/spdx/license-list-data/tarball"
+		githubApiToken            = ""
+		githubUsername            = ""
+		destination               = ""
+		licenseAliasesDestination = ""
 	)
 
 	flag.StringVar(&destination, "dest", "options/license/", "destination for the licenses")
+	flag.StringVar(&licenseAliasesDestination, "aliasesdest", "options/license-aliases.json", "destination for same license json")
 	flag.StringVar(&githubUsername, "username", "", "github username")
 	flag.StringVar(&githubApiToken, "token", "", "github api token")
 	flag.Parse()
@@ -77,7 +82,7 @@ func main() {
 	}
 
 	tr := tar.NewReader(gz)
-
+	aliasesFiles := make(map[string][]string)
 	for {
 		hdr, err := tr.Next()
 
@@ -97,26 +102,97 @@ func main() {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "README") {
+		fileBaseName := filepath.Base(hdr.Name)
+		licenseName := strings.TrimSuffix(fileBaseName, ".txt")
+
+		if strings.HasPrefix(fileBaseName, "README") {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") {
+		if strings.HasPrefix(fileBaseName, "deprecated_") {
 			continue
 		}
-		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt")))
+		out, err := os.Create(path.Join(destination, licenseName))
 		if err != nil {
 			log.Fatalf("Failed to create new file. %s", err)
 		}
 
 		defer out.Close()
 
-		if _, err := io.Copy(out, tr); err != nil {
+		// some license files have same content, so we need to detect these files and create a convert map into a json file
+		// Later we use this convert map to avoid adding same license content with different license name
+		h := md5.New()
+		// calculate md5 and write file in the same time
+		r := io.TeeReader(tr, h)
+		if _, err := io.Copy(out, r); err != nil {
 			log.Fatalf("Failed to write new file. %s", err)
 		} else {
 			fmt.Printf("Written %s\n", out.Name())
+
+			md5 := hex.EncodeToString(h.Sum(nil))
+			aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
+		}
+	}
+
+	// generate convert license name map
+	licenseAliases := make(map[string]string)
+	for _, fileNames := range aliasesFiles {
+		if len(fileNames) > 1 {
+			licenseName := getLicenseNameFromAliases(fileNames)
+			for _, fileName := range fileNames {
+				licenseAliases[fileName] = licenseName
+			}
 		}
 	}
+	// save convert license name map to file
+	b, err := json.Marshal(licenseAliases)
+	if err != nil {
+		log.Fatalf("Failed to create json bytes. %s", err)
+		return
+	}
+	f, err := os.Create(licenseAliasesDestination)
+	if err != nil {
+		log.Fatalf("Failed to create license aliases json file. %s", err)
+	}
+	defer f.Close()
+	if _, err = f.Write(b); err != nil {
+		log.Fatalf("Failed to write license aliases json file. %s", err)
+	}
 
 	fmt.Println("Done")
 }
+
+func getLicenseNameFromAliases(fnl []string) string {
+	if len(fnl) == 0 {
+		return ""
+	}
+
+	shortestItem := func(list []string) string {
+		s := list[0]
+		for _, l := range list[1:] {
+			if len(l) < len(s) {
+				s = l
+			}
+		}
+		return s
+	}
+	allHasPrefix := func(list []string, s string) bool {
+		for _, l := range list {
+			if !strings.HasPrefix(l, s) {
+				return false
+			}
+		}
+		return true
+	}
+
+	sl := shortestItem(fnl)
+	slv := strings.Split(sl, "-")
+	var result string
+	for i := len(slv); i >= 0; i-- {
+		result = strings.Join(slv[:i], "-")
+		if allHasPrefix(fnl, result) {
+			return result
+		}
+	}
+	return ""
+}

go.mod

@@ -56,6 +56,7 @@ require (
 	github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
 	github.com/golang-jwt/jwt/v5 v5.2.0
 	github.com/google/go-github/v57 v57.0.0
+	github.com/google/licenseclassifier/v2 v2.0.0
 	github.com/google/pprof v0.0.0-20240117000934-35fc243c5815
 	github.com/google/uuid v1.5.0
 	github.com/gorilla/feeds v1.1.2

go.sum

@@ -471,6 +471,8 @@ github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=
+github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -786,6 +788,7 @@ github.com/sassoftware/go-rpmutils v0.2.1-0.20240124161140-277b154961dd/go.mod h
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

models/fixtures/repo_license.yml

@@ -0,0 +1 @@
+[] # empty

models/fixtures/repository.yml

@@ -26,7 +26,7 @@
   fork_id: 0
   is_template: false
   template_id: 0
-  size: 7320
+  size: 8201
   is_fsck_enabled: true
   close_issues_via_commit_in_any_branch: false
 

models/migrations/migrations.go

@@ -499,7 +499,7 @@ var migrations = []Migration{
 	// v259 -> v260
 	NewMigration("Convert scoped access tokens", v1_20.ConvertScopedAccessTokens),
 
-	// Gitea 1.20.0 ends at 260
+	// Gitea 1.20.0 ends at v260
 
 	// v260 -> v261
 	NewMigration("Drop custom_labels column of action_runner table", v1_21.DropCustomLabelsColumnOfActionRunner),
@@ -562,6 +562,8 @@ var migrations = []Migration{
 	NewMigration("Use Slug instead of ID for Badges", v1_22.UseSlugInsteadOfIDForBadges),
 	// v288 -> v289
 	NewMigration("Add user_blocking table", v1_22.AddUserBlockingTable),
+	// v289 -> v290
+	NewMigration("Add Repository Licenses", v1_22.AddRepositoryLicenses),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_22/v289.go

@@ -0,0 +1,23 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_22 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func AddRepositoryLicenses(x *xorm.Engine) error {
+	type RepoLicense struct {
+		ID          int64 `xorm:"pk autoincr"`
+		RepoID      int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		CommitID    string
+		License     string             `xorm:"VARCHAR(50) UNIQUE(s) INDEX NOT NULL"`
+		CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+		UpdatedUnix timeutil.TimeStamp `xorm:"INDEX UPDATED"`
+	}
+
+	return x.Sync(new(RepoLicense))
+}

models/repo/license.go

@@ -0,0 +1,114 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repo
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+func init() {
+	db.RegisterModel(new(RepoLicense))
+}
+
+type RepoLicense struct { //revive:disable-line:exported
+	ID       int64 `xorm:"pk autoincr"`
+	RepoID   int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	CommitID string
+	// TODO: `INDEX` will be used to find repositories by license
+	License     string             `xorm:"VARCHAR(50) UNIQUE(s) INDEX NOT NULL"`
+	CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX UPDATED"`
+}
+
+// RepoLicenseList defines a list of repo licenses
+type RepoLicenseList []*RepoLicense //revive:disable-line:exported
+
+func (rll RepoLicenseList) StringList() []string {
+	var licenses []string
+	for _, rl := range rll {
+		licenses = append(licenses, rl.License)
+	}
+	return licenses
+}
+
+// GetRepoLicenses returns the license statistics for a repository
+func GetRepoLicenses(ctx context.Context, repo *Repository) (RepoLicenseList, error) {
+	licenses := make(RepoLicenseList, 0)
+	if err := db.GetEngine(ctx).Where("`repo_id` = ?", repo.ID).Asc("`license`").Find(&licenses); err != nil {
+		return nil, err
+	}
+	return licenses, nil
+}
+
+// UpdateRepoLicenses updates the license statistics for repository
+func UpdateRepoLicenses(ctx context.Context, repo *Repository, commitID string, licenses []string) error {
+	oldLicenses, err := GetRepoLicenses(ctx, repo)
+	if err != nil {
+		return err
+	}
+	for _, license := range licenses {
+		upd := false
+		for _, o := range oldLicenses {
+			// Update already existing license
+			if o.License == license {
+				if _, err := db.GetEngine(ctx).ID(o.ID).Cols("`commit_id`").Update(o); err != nil {
+					return err
+				}
+				upd = true
+				break
+			}
+		}
+		// Insert new license
+		if !upd {
+			if err := db.Insert(ctx, &RepoLicense{
+				RepoID:   repo.ID,
+				CommitID: commitID,
+				License:  license,
+			}); err != nil {
+				return err
+			}
+		}
+	}
+	// Delete old licenses
+	licenseToDelete := make([]int64, 0, len(oldLicenses))
+	for _, o := range oldLicenses {
+		if o.CommitID != commitID {
+			licenseToDelete = append(licenseToDelete, o.ID)
+		}
+	}
+	if len(licenseToDelete) > 0 {
+		if _, err := db.GetEngine(ctx).In("`id`", licenseToDelete).Delete(&RepoLicense{}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// CopyLicense Copy originalRepo license information to destRepo (use for forked repo)
+func CopyLicense(ctx context.Context, originalRepo, destRepo *Repository) error {
+	repoLicenses, err := GetRepoLicenses(ctx, originalRepo)
+	if err != nil {
+		return err
+	}
+	if len(repoLicenses) > 0 {
+		newRepoLicenses := make(RepoLicenseList, 0, len(repoLicenses))
+
+		for _, rl := range repoLicenses {
+			newRepoLicense := &RepoLicense{
+				RepoID:   destRepo.ID,
+				CommitID: rl.CommitID,
+				License:  rl.License,
+			}
+			newRepoLicenses = append(newRepoLicenses, newRepoLicense)
+		}
+		if err := db.Insert(ctx, &newRepoLicenses); err != nil {
+			return err
+		}
+	}
+	return nil
+}

modules/repository/license.go

@@ -23,7 +23,7 @@ type LicenseValues struct {
 func GetLicense(name string, values *LicenseValues) ([]byte, error) {
 	data, err := options.License(name)
 	if err != nil {
-		return nil, fmt.Errorf("GetRepoInitFile[%s]: %w", name, err)
+		return nil, fmt.Errorf("GetLicense[%s]: %w", name, err)
 	}
 	return fillLicensePlaceholder(name, values, data), nil
 }

modules/structs/repo.go

@@ -113,6 +113,7 @@ type Repository struct {
 	// swagger:strfmt date-time
 	MirrorUpdated time.Time     `json:"mirror_updated,omitempty"`
 	RepoTransfer  *RepoTransfer `json:"repo_transfer"`
+	Licenses      []string      `json:"licenses"`
 }
 
 // CreateRepoOption options when creating repository