From e833dbd61be24233dc8221a0fcc43ae03ba043c0 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 30 Apr 2023 21:24:42 +0000 Subject: [PATCH 1/3] Implement Cargo HTTP index. --- routers/api/packages/api.go | 5 + routers/api/packages/cargo/cargo.go | 30 +++++ services/packages/cargo/index.go | 36 ++++-- tests/integration/api_packages_cargo_test.go | 112 +++++++++++++------ 4 files changed, 140 insertions(+), 43 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index d5acd3d261165..acc75bac41833 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -118,6 +118,11 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { r.Get("/owners", cargo.ListOwners) }) }) + r.Get("/config.json", cargo.RepositoryConfig) + r.Get("/1/{package}", cargo.EnumeratePackageVersions) + r.Get("/2/{package}", cargo.EnumeratePackageVersions) + r.Get("/3/{_}/{package}", cargo.EnumeratePackageVersions) + r.Get("/{_}/{__}/{package}", cargo.EnumeratePackageVersions) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/chef", func() { r.Group("/api/v1", func() { diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index e0bf5da13adb0..be1270297654c 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -4,6 +4,7 @@ package cargo import ( + "errors" "fmt" "net/http" "strconv" @@ -45,6 +46,35 @@ func apiError(ctx *context.Context, status int, obj interface{}) { }) } +// https://rust-lang.github.io/rfcs/2789-sparse-index.html +func RepositoryConfig(ctx *context.Context) { + ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner)) +} + +func EnumeratePackageVersions(ctx *context.Context) { + p, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package")) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + b, err := cargo_service.BuildPackageIndex(ctx, p) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if b == nil { + apiError(ctx, http.StatusNotFound, nil) + return + } + + ctx.PlainTextBytes(http.StatusOK, b.Bytes()) +} + type SearchResult struct { Crates []*SearchResultCrate `json:"crates"` Meta SearchResultMeta `json:"meta"` diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index e58a47281628b..1c2d17b504fc3 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -137,21 +137,21 @@ type IndexVersionEntry struct { Links string `json:"links,omitempty"` } -func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { +func BuildPackageIndex(ctx context.Context, p *packages_model.Package) (*bytes.Buffer, error) { pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ PackageID: p.ID, Sort: packages_model.SortVersionAsc, }) if err != nil { - return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) + return nil, fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) } if len(pvs) == 0 { - return nil + return nil, nil } pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { - return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) + return nil, fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) } var b bytes.Buffer @@ -179,14 +179,26 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo Links: metadata.Links, }) if err != nil { - return err + return nil, err } b.Write(entry) b.WriteString("\n") } - return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b) + return &b, nil +} + +func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { + b, err := BuildPackageIndex(ctx, p) + if err != nil { + return err + } + if b == nil { + return nil + } + + return writeObjectToIndex(t, BuildPackagePath(p.LowerName), b) } func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { @@ -212,6 +224,13 @@ type Config struct { APIURL string `json:"api"` } +func BuildConfig(owner *user_model.User) *Config { + return &Config{ + DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", + APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", + } +} + func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error { return alterRepositoryContent( ctx, @@ -220,10 +239,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, "Initialize Cargo Config", func(t *files_service.TemporaryUploadRepository) error { var b bytes.Buffer - err := json.NewEncoder(&b).Encode(Config{ - DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", - APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", - }) + err := json.NewEncoder(&b).Encode(BuildConfig(owner)) if err != nil { return err } diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index 0c542eaf1e857..f8fb35041141d 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -98,7 +98,7 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { url := fmt.Sprintf("%s/api/v1/crates", root) t.Run("Index", func(t *testing.T) { - t.Run("Config", func(t *testing.T) { + t.Run("Git/Config", func(t *testing.T) { defer tests.PrintCurrentTest(t)() content := readGitContent(t, cargo_service.ConfigFileName) @@ -110,6 +110,20 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { assert.Equal(t, url, config.DownloadURL) assert.Equal(t, root, config.APIURL) }) + + t.Run("HTTP/Config", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", root+"/"+cargo_service.ConfigFileName) + resp := MakeRequest(t, req, http.StatusOK) + + var config cargo_service.Config + err := json.Unmarshal(resp.Body.Bytes(), &config) + assert.NoError(t, err) + + assert.Equal(t, url, config.DownloadURL) + assert.Equal(t, root, config.APIURL) + }) }) t.Run("Upload", func(t *testing.T) { @@ -192,40 +206,72 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { MakeRequest(t, req, http.StatusConflict) t.Run("Index", func(t *testing.T) { - t.Run("Entry", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) - - var entry cargo_service.IndexVersionEntry - err := json.Unmarshal([]byte(content), &entry) - assert.NoError(t, err) - - assert.Equal(t, packageName, entry.Name) - assert.Equal(t, packageVersion, entry.Version) - assert.Equal(t, pb.HashSHA256, entry.FileChecksum) - assert.False(t, entry.Yanked) - assert.Len(t, entry.Dependencies, 1) - dep := entry.Dependencies[0] - assert.Equal(t, "dep", dep.Name) - assert.Equal(t, "1.0", dep.Req) - assert.Equal(t, "normal", dep.Kind) - assert.True(t, dep.DefaultFeatures) - assert.Empty(t, dep.Features) - assert.False(t, dep.Optional) - assert.Nil(t, dep.Target) - assert.NotNil(t, dep.Registry) - assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) - assert.Nil(t, dep.Package) + t.Run("Git", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) + }) + + t.Run("Rebuild", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + err := cargo_service.RebuildIndex(db.DefaultContext, user, user) + assert.NoError(t, err) + + _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + }) }) - t.Run("Rebuild", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - err := cargo_service.RebuildIndex(db.DefaultContext, user, user) - assert.NoError(t, err) - - _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + t.Run("HTTP", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", root+"/"+cargo_service.BuildPackagePath(packageName)) + resp := MakeRequest(t, req, http.StatusOK) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal(resp.Body.Bytes(), &entry) + assert.NoError(t, err) + + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) + }) }) }) }) From 692b1cad965dfbaff076a7be7ac85a0b6cc60f13 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 30 Apr 2023 21:57:00 +0000 Subject: [PATCH 2/3] lint --- tests/integration/api_packages_cargo_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index f8fb35041141d..608f19296819f 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -209,13 +209,13 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { t.Run("Git", func(t *testing.T) { t.Run("Entry", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) - + var entry cargo_service.IndexVersionEntry err := json.Unmarshal([]byte(content), &entry) assert.NoError(t, err) - + assert.Equal(t, packageName, entry.Name) assert.Equal(t, packageVersion, entry.Version) assert.Equal(t, pb.HashSHA256, entry.FileChecksum) @@ -233,13 +233,13 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) assert.Nil(t, dep.Package) }) - + t.Run("Rebuild", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - + err := cargo_service.RebuildIndex(db.DefaultContext, user, user) assert.NoError(t, err) - + _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) }) }) @@ -250,11 +250,11 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { req := NewRequest(t, "GET", root+"/"+cargo_service.BuildPackagePath(packageName)) resp := MakeRequest(t, req, http.StatusOK) - + var entry cargo_service.IndexVersionEntry err := json.Unmarshal(resp.Body.Bytes(), &entry) assert.NoError(t, err) - + assert.Equal(t, packageName, entry.Name) assert.Equal(t, packageVersion, entry.Version) assert.Equal(t, pb.HashSHA256, entry.FileChecksum) From 059c647b564d6afa6fd6d617d2555e8f8ad3ee43 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 3 May 2023 06:52:36 +0000 Subject: [PATCH 3/3] Add placeholder comment. --- routers/api/packages/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index d366c7dc69b09..ee1feb1414c1c 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -122,6 +122,7 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { r.Get("/config.json", cargo.RepositoryConfig) r.Get("/1/{package}", cargo.EnumeratePackageVersions) r.Get("/2/{package}", cargo.EnumeratePackageVersions) + // Use dummy placeholders because these parts are not of interest r.Get("/3/{_}/{package}", cargo.EnumeratePackageVersions) r.Get("/{_}/{__}/{package}", cargo.EnumeratePackageVersions) }, reqPackageAccess(perm.AccessModeRead))