publish: Use sync_to_git/sparse_index() jobs if FEATURE_INDEX_SYNC env var is set

Turbo87 · Turbo87 · commit ef234a43b05c · 2023-04-21T18:35:53.000+02:00
diff --git a/src/config.rs b/src/config.rs
@@ -47,6 +47,7 @@ pub struct Server {
     pub version_id_cache_ttl: Duration,
     pub cdn_user_agent: String,
     pub balance_capacity: BalanceCapacityConfig,
+    pub feature_index_sync: bool,
 }
 
 impl Default for Server {
@@ -151,6 +152,7 @@ impl Default for Server {
             cdn_user_agent: dotenv::var("WEB_CDN_USER_AGENT")
                 .unwrap_or_else(|_| "Amazon CloudFront".into()),
             balance_capacity: BalanceCapacityConfig::from_environment(),
+            feature_index_sync: dotenv::var("FEATURE_INDEX_SYNC").is_ok(),
         }
     }
 }
diff --git a/src/controllers/krate/publish.rs b/src/controllers/krate/publish.rs
@@ -1,6 +1,7 @@
 //! Functionality related to publishing a new crate or version of a crate.
 
 use crate::auth::AuthCheck;
+use crate::background_jobs::Job;
 use axum::body::Bytes;
 use flate2::read::GzDecoder;
 use hex::ToHex;
@@ -271,7 +272,13 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
                 links,
                 v,
             };
-            worker::add_crate(git_crate).enqueue(conn)?;
+
+            if app.config.feature_index_sync {
+                Job::sync_to_git_index(&git_crate.name).enqueue(conn)?;
+                Job::sync_to_sparse_index(&git_crate.name).enqueue(conn)?;
+            } else {
+                worker::add_crate(git_crate).enqueue(conn)?;
+            }
 
             // The `other` field on `PublishWarnings` was introduced to handle a temporary warning
             // that is no longer needed. As such, crates.io currently does not return any `other`
diff --git a/src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted.json b/src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted.json
@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "154"
+          "309"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "{\"name\":\"foo_whitelist\",\"vers\":\"1.1.0\",\"deps\":[],\"cksum\":\"4e33dc59bbbc96645fc01945fb502507d1b7bd3a2d06227bf7b0fe8842f284c2\",\"features\":{},\"yanked\":false}\n"
+      "body": "{\"name\":\"foo_whitelist\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\"                                                                \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_whitelist\",\"vers\":\"1.1.0\",\"deps\":[],\"cksum\":\"4e33dc59bbbc96645fc01945fb502507d1b7bd3a2d06227bf7b0fe8842f284c2\",\"features\":{},\"yanked\":false}\n"
     },
     "response": {
       "status": 200,
diff --git a/src/tests/http-data/krate_publish_new_krate_twice.json b/src/tests/http-data/krate_publish_new_krate_twice.json
@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "150"
+          "301"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "{\"name\":\"foo_twice\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
+      "body": "{\"name\":\"foo_twice\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\"                                                                \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_twice\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
     },
     "response": {
       "status": 200,
diff --git a/src/tests/http-data/krate_publish_publish_after_removing_documentation.json b/src/tests/http-data/krate_publish_publish_after_removing_documentation.json
@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "150"
+          "300"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
+      "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.0\",\"deps\":[],\"cksum\":\"                                                                \",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
     },
     "response": {
       "status": 200,
@@ -104,14 +104,14 @@
         ],
         [
           "content-length",
-          "300"
+          "450"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.2\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
+      "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.0\",\"deps\":[],\"cksum\":\"                                                                \",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.2\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
     },
     "response": {
       "status": 200,
diff --git a/src/tests/http-data/team_publish_owned.json b/src/tests/http-data/team_publish_owned.json
@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "155"
+          "311"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "{\"name\":\"foo_team_owned\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
+      "body": "{\"name\":\"foo_team_owned\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\"                                                                \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_team_owned\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n"
     },
     "response": {
       "status": 200,
diff --git a/src/tests/util/test_app.rs b/src/tests/util/test_app.rs
@@ -359,6 +359,7 @@ fn simple_config() -> config::Server {
         version_id_cache_ttl: Duration::from_secs(5 * 60),
         cdn_user_agent: "Amazon CloudFront".to_string(),
         balance_capacity: BalanceCapacityConfig::for_testing(),
+        feature_index_sync: true,
     }
 }
 

Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,7 @@ pub struct Server {`
`47`	`47`	`pub version_id_cache_ttl: Duration,`
`48`	`48`	`pub cdn_user_agent: String,`
`49`	`49`	`pub balance_capacity: BalanceCapacityConfig,`
	`50`	`+ pub feature_index_sync: bool,`
`50`	`51`	`}`
`51`	`52`
`52`	`53`	`impl Default for Server {`
`@@ -151,6 +152,7 @@ impl Default for Server {`
`151`	`152`	`cdn_user_agent: dotenv::var("WEB_CDN_USER_AGENT")`
`152`	`153`	`.unwrap_or_else(\|_\| "Amazon CloudFront".into()),`
`153`	`154`	`balance_capacity: BalanceCapacityConfig::from_environment(),`
	`155`	`+ feature_index_sync: dotenv::var("FEATURE_INDEX_SYNC").is_ok(),`
`154`	`156`	`}`
`155`	`157`	`}`
`156`	`158`	`}`
Original file line number	Diff line number	Diff line change
`@@ -44,14 +44,14 @@`
`44`	`44`	`],`
`45`	`45`	`[`
`46`	`46`	`"content-length",`
`47`		`- "154"`
	`47`	`+ "309"`
`48`	`48`	`],`
`49`	`49`	`[`
`50`	`50`	`"content-type",`
`51`	`51`	`"text/plain"`
`52`	`52`	`]`
`53`	`53`	`],`
`54`		`- "body": "{\"name\":\"foo_whitelist\",\"vers\":\"1.1.0\",\"deps\":[],\"cksum\":\"4e33dc59bbbc96645fc01945fb502507d1b7bd3a2d06227bf7b0fe8842f284c2\",\"features\":{},\"yanked\":false}\n"`
	`54`	`+ "body": "{\"name\":\"foo_whitelist\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\" \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_whitelist\",\"vers\":\"1.1.0\",\"deps\":[],\"cksum\":\"4e33dc59bbbc96645fc01945fb502507d1b7bd3a2d06227bf7b0fe8842f284c2\",\"features\":{},\"yanked\":false}\n"`
`55`	`55`	`},`
`56`	`56`	`"response": {`
`57`	`57`	`"status": 200,`
Original file line number	Diff line number	Diff line change
`@@ -359,6 +359,7 @@ fn simple_config() -> config::Server {`
`359`	`359`	`version_id_cache_ttl: Duration::from_secs(5 * 60),`
`360`	`360`	`cdn_user_agent: "Amazon CloudFront".to_string(),`
`361`	`361`	`balance_capacity: BalanceCapacityConfig::for_testing(),`
	`362`	`+ feature_index_sync: true,`
`362`	`363`	`}`
`363`	`364`	`}`
`364`	`365`