Support repo license (#24872)

Close #278
Close #24076

## Solutions:
- Use
[google/licenseclassifier](https://github.com/google/licenseclassifier/)
Test result between
[google/licensecheck](https://github.com/google/licensecheck) and
[go-license-detector](https://github.com/go-enry/go-license-detector):
#24872 (comment)
Test result between
[google/licensecheck](https://github.com/google/licensecheck) and
[google/licenseclassifier](https://github.com/google/licenseclassifier/):
#24872 (comment)
- Generate License Convert Name List to avoid import license templates
with same contents
Gitea automatically get latest license data from[
spdx/license-list-data](https://github.com/spdx/license-list-data).
But unfortunately, some license templates have same contents. #20915
[click here to see the
list](#24872 (comment))
So we will generate a list of these license templates with same contents
and create a new file to save the result when using `make
generate-license`. (Need to decide the save path)
- Save License info into a new table `repo_license`
Can easily support searching repo by license in the future.

## Screen shot
Single License:

![image](https://github.com/go-gitea/gitea/assets/18380374/41260bd7-0b4c-4038-8592-508706cffa9f)

Multiple Licenses:

![image](https://github.com/go-gitea/gitea/assets/18380374/34ce2f73-7e18-446b-9b96-ecc4fb61bd70)

Triggers:
- [x] Push commit to default branch
- [x] Create repo
- [x] Mirror repo
- [x] When Default Branch is changed, licenses should be updated

Todo:
- [x] Save Licenses info in to DB when there's a change to license file
in the commit
- [x] DB Migration
- [x] A nominal test?
- [x] Select which library to
use(#24872 (comment))
- [x] API Support
- [x] Add repo license table
- ~Select license in settings if there are several licenses(Not
recommended)~
- License board(later, not in this PR)

![image](https://github.com/go-gitea/gitea/assets/18380374/2c3c3bf8-bcc2-4c6d-8ce0-81d1a9733878)

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: Denys Konovalov <kontakt@denyskon.de>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: KN4CK3R <admin@oldschoolhack.me>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: 6543 <m.huber@kithara.com>
Co-authored-by: a1012112796 <1012112796@qq.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.com>

Author: 9 people

assets/go-licenses.json

@@ -1,10 +1,15 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
 //go:build ignore
 
 package main
 
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/md5"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -15,6 +20,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"code.gitea.io/gitea/build/license"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/util"
 )
 
@@ -77,7 +84,7 @@ func main() {
 	}
 
 	tr := tar.NewReader(gz)
-
+	aliasesFiles := make(map[string][]string)
 	for {
 		hdr, err := tr.Next()
 
@@ -97,26 +104,73 @@ func main() {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "README") {
+		fileBaseName := filepath.Base(hdr.Name)
+		licenseName := strings.TrimSuffix(fileBaseName, ".txt")
+
+		if strings.HasPrefix(fileBaseName, "README") {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") {
+		if strings.HasPrefix(fileBaseName, "deprecated_") {
 			continue
 		}
-		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt")))
+		out, err := os.Create(path.Join(destination, licenseName))
 		if err != nil {
 			log.Fatalf("Failed to create new file. %s", err)
 		}
 
 		defer out.Close()
 
-		if _, err := io.Copy(out, tr); err != nil {
+		// some license files have same content, so we need to detect these files and create a convert map into a json file
+		// Later we use this convert map to avoid adding same license content with different license name
+		h := md5.New()
+		// calculate md5 and write file in the same time
+		r := io.TeeReader(tr, h)
+		if _, err := io.Copy(out, r); err != nil {
 			log.Fatalf("Failed to write new file. %s", err)
 		} else {
 			fmt.Printf("Written %s\n", out.Name())
+
+			md5 := hex.EncodeToString(h.Sum(nil))
+			aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
 		}
 	}
 
+	// generate convert license name map
+	licenseAliases := make(map[string]string)
+	for _, fileNames := range aliasesFiles {
+		if len(fileNames) > 1 {
+			licenseName := license.GetLicenseNameFromAliases(fileNames)
+			if licenseName == "" {
+				// license name should not be empty as expected
+				// if it is empty, we need to rewrite the logic of GetLicenseNameFromAliases
+				log.Fatalf("GetLicenseNameFromAliases: license name is empty")
+			}
+			for _, fileName := range fileNames {
+				licenseAliases[fileName] = licenseName
+			}
+		}
+	}
+	// save convert license name map to file
+	b, err := json.Marshal(licenseAliases)
+	if err != nil {
+		log.Fatalf("Failed to create json bytes. %s", err)
+	}
+
+	licenseAliasesDestination := filepath.Join(destination, "etc", "license-aliases.json")
+	if err := os.MkdirAll(filepath.Dir(licenseAliasesDestination), 0o755); err != nil {
+		log.Fatalf("Failed to create directory for license aliases json file. %s", err)
+	}
+
+	f, err := os.Create(licenseAliasesDestination)
+	if err != nil {
+		log.Fatalf("Failed to create license aliases json file. %s", err)
+	}
+	defer f.Close()
+
+	if _, err = f.Write(b); err != nil {
+		log.Fatalf("Failed to write license aliases json file. %s", err)
+	}
+
 	fmt.Println("Done")
 }

build/generate-licenses.go

@@ -0,0 +1,41 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import "strings"
+
+func GetLicenseNameFromAliases(fnl []string) string {
+	if len(fnl) == 0 {
+		return ""
+	}
+
+	shortestItem := func(list []string) string {
+		s := list[0]
+		for _, l := range list[1:] {
+			if len(l) < len(s) {
+				s = l
+			}
+		}
+		return s
+	}
+	allHasPrefix := func(list []string, s string) bool {
+		for _, l := range list {
+			if !strings.HasPrefix(l, s) {
+				return false
+			}
+		}
+		return true
+	}
+
+	sl := shortestItem(fnl)
+	slv := strings.Split(sl, "-")
+	var result string
+	for i := len(slv); i >= 0; i-- {
+		result = strings.Join(slv[:i], "-")
+		if allHasPrefix(fnl, result) {
+			return result
+		}
+	}
+	return ""
+}

build/license/aliasgenerator.go

@@ -0,0 +1,39 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetLicenseNameFromAliases(t *testing.T) {
+	tests := []struct {
+		target string
+		inputs []string
+	}{
+		{
+			// real case which you can find in license-aliases.json
+			target: "AGPL-1.0",
+			inputs: []string{
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+		{
+			target: "",
+			inputs: []string{
+				"APSL-1.0",
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		result := GetLicenseNameFromAliases(tt.inputs)
+		assert.Equal(t, result, tt.target)
+	}
+}

build/license/aliasgenerator_test.go

@@ -68,6 +68,7 @@ require (
 	github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/go-github/v61 v61.0.0
+	github.com/google/licenseclassifier/v2 v2.0.0
 	github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/feeds v1.2.0

go.mod

@@ -441,6 +441,8 @@ github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=
+github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM=
 github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 h1:ASJ/LAqdCHOyMYI+dwNxn7Rd8FscNkMyTr1KZU1JI/M=
 github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
@@ -735,6 +737,7 @@ github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jN
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

go.sum

@@ -0,0 +1 @@
+[] # empty

models/fixtures/repo_license.yml

@@ -26,7 +26,7 @@
   fork_id: 0
   is_template: false
   template_id: 0
-  size: 7597
+  size: 8478
   is_fsck_enabled: true
   close_issues_via_commit_in_any_branch: false
 

models/fixtures/repository.yml

@@ -500,7 +500,7 @@ var migrations = []Migration{
 	// v259 -> v260
 	NewMigration("Convert scoped access tokens", v1_20.ConvertScopedAccessTokens),
 
-	// Gitea 1.20.0 ends at 260
+	// Gitea 1.20.0 ends at v260
 
 	// v260 -> v261
 	NewMigration("Drop custom_labels column of action_runner table", v1_21.DropCustomLabelsColumnOfActionRunner),
@@ -601,6 +601,8 @@ var migrations = []Migration{
 	NewMigration("Add metadata column for comment table", v1_23.AddCommentMetaDataColumn),
 	// v304 -> v305
 	NewMigration("Add index for release sha1", v1_23.AddIndexForReleaseSha1),
+	// v305 -> v306
+	NewMigration("Add Repository Licenses", v1_23.AddRepositoryLicenses),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/migrations.go

@@ -0,0 +1,23 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_23 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func AddRepositoryLicenses(x *xorm.Engine) error {
+	type RepoLicense struct {
+		ID          int64 `xorm:"pk autoincr"`
+		RepoID      int64 `xorm:"UNIQUE(s) NOT NULL"`
+		CommitID    string
+		License     string             `xorm:"VARCHAR(255) UNIQUE(s) NOT NULL"`
+		CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+		UpdatedUnix timeutil.TimeStamp `xorm:"INDEX UPDATED"`
+	}
+
+	return x.Sync(new(RepoLicense))
+}

models/migrations/v1_23/v305.go

@@ -0,0 +1,120 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repo
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+func init() {
+	db.RegisterModel(new(RepoLicense))
+}
+
+type RepoLicense struct { //revive:disable-line:exported
+	ID          int64 `xorm:"pk autoincr"`
+	RepoID      int64 `xorm:"UNIQUE(s) NOT NULL"`
+	CommitID    string
+	License     string             `xorm:"VARCHAR(255) UNIQUE(s) NOT NULL"`
+	CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX UPDATED"`
+}
+
+// RepoLicenseList defines a list of repo licenses
+type RepoLicenseList []*RepoLicense //revive:disable-line:exported
+
+func (rll RepoLicenseList) StringList() []string {
+	var licenses []string
+	for _, rl := range rll {
+		licenses = append(licenses, rl.License)
+	}
+	return licenses
+}
+
+// GetRepoLicenses returns the license statistics for a repository
+func GetRepoLicenses(ctx context.Context, repo *Repository) (RepoLicenseList, error) {
+	licenses := make(RepoLicenseList, 0)
+	if err := db.GetEngine(ctx).Where("`repo_id` = ?", repo.ID).Asc("`license`").Find(&licenses); err != nil {
+		return nil, err
+	}
+	return licenses, nil
+}
+
+// UpdateRepoLicenses updates the license statistics for repository
+func UpdateRepoLicenses(ctx context.Context, repo *Repository, commitID string, licenses []string) error {
+	oldLicenses, err := GetRepoLicenses(ctx, repo)
+	if err != nil {
+		return err
+	}
+	for _, license := range licenses {
+		upd := false
+		for _, o := range oldLicenses {
+			// Update already existing license
+			if o.License == license {
+				if _, err := db.GetEngine(ctx).ID(o.ID).Cols("`commit_id`").Update(o); err != nil {
+					return err
+				}
+				upd = true
+				break
+			}
+		}
+		// Insert new license
+		if !upd {
+			if err := db.Insert(ctx, &RepoLicense{
+				RepoID:   repo.ID,
+				CommitID: commitID,
+				License:  license,
+			}); err != nil {
+				return err
+			}
+		}
+	}
+	// Delete old licenses
+	licenseToDelete := make([]int64, 0, len(oldLicenses))
+	for _, o := range oldLicenses {
+		if o.CommitID != commitID {
+			licenseToDelete = append(licenseToDelete, o.ID)
+		}
+	}
+	if len(licenseToDelete) > 0 {
+		if _, err := db.GetEngine(ctx).In("`id`", licenseToDelete).Delete(&RepoLicense{}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// CopyLicense Copy originalRepo license information to destRepo (use for forked repo)
+func CopyLicense(ctx context.Context, originalRepo, destRepo *Repository) error {
+	repoLicenses, err := GetRepoLicenses(ctx, originalRepo)
+	if err != nil {
+		return err
+	}
+	if len(repoLicenses) > 0 {
+		newRepoLicenses := make(RepoLicenseList, 0, len(repoLicenses))
+
+		for _, rl := range repoLicenses {
+			newRepoLicense := &RepoLicense{
+				RepoID:   destRepo.ID,
+				CommitID: rl.CommitID,
+				License:  rl.License,
+			}
+			newRepoLicenses = append(newRepoLicenses, newRepoLicense)
+		}
+		if err := db.Insert(ctx, &newRepoLicenses); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CleanRepoLicenses will remove all license record of the repo
+func CleanRepoLicenses(ctx context.Context, repo *Repository) error {
+	return db.DeleteBeans(ctx, &RepoLicense{
+		RepoID: repo.ID,
+	})
+}