diff --git a/Cargo.lock b/Cargo.lock index ed2f09a4829..941e1a8349f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,6 +513,7 @@ version = "0.0.0" dependencies = [ "anyhow", "base64 0.13.1", + "claims", "dotenv", "git2", "serde", diff --git a/cargo-registry-index/Cargo.toml b/cargo-registry-index/Cargo.toml index a76fcbbdd35..5b82529ab77 100644 --- a/cargo-registry-index/Cargo.toml +++ b/cargo-registry-index/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" path = "lib.rs" [features] -testing = ["serde_json"] +testing = [] [dependencies] anyhow = "=1.0.70" @@ -18,7 +18,10 @@ base64 = "=0.13.1" dotenv = "=0.15.0" git2 = "=0.17.1" serde = { version = "=1.0.160", features = ["derive"] } +serde_json = "=1.0.96" tempfile = "=3.5.0" tracing = "=0.1.37" url = "=2.3.1" -serde_json = { version = "=1.0.96", optional = true } + +[dev-dependencies] +claims = "=0.7.1" diff --git a/cargo-registry-index/lib.rs b/cargo-registry-index/lib.rs index 5050e1b7810..6e0db04c871 100644 --- a/cargo-registry-index/lib.rs +++ b/cargo-registry-index/lib.rs @@ -148,6 +148,21 @@ pub struct Crate { pub v: Option, } +impl Crate { + pub fn write_to(&self, mut writer: W) -> anyhow::Result<()> { + serde_json::to_writer(&mut writer, self)?; + writer.write_all(b"\n")?; + Ok(()) + } +} + +pub fn write_crates(crates: &[Crate], mut writer: W) -> anyhow::Result<()> { + for krate in crates { + krate.write_to(&mut writer)?; + } + Ok(()) +} + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] pub struct Dependency { pub name: String, @@ -595,3 +610,57 @@ pub fn run_via_cli(command: &mut Command, credentials: &Credentials) -> anyhow:: Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use claims::*; + + #[test] + fn crate_writer() { + let krate = Crate { + name: "foo".to_string(), + vers: "1.2.3".to_string(), + deps: vec![], + cksum: "0123456789asbcdef".to_string(), + features: Default::default(), + features2: None, + yanked: None, + links: None, + v: None, + }; + let mut buffer = Vec::new(); + assert_ok!(krate.write_to(&mut buffer)); + assert_ok_eq!(String::from_utf8(buffer), "\ + {\"name\":\"foo\",\"vers\":\"1.2.3\",\"deps\":[],\"cksum\":\"0123456789asbcdef\",\"features\":{},\"yanked\":null}\n\ + "); + } + + #[test] + fn test_write_crates() { + let versions = vec!["0.1.0", "1.0.0-beta.1", "1.0.0", "1.2.3"]; + let crates = versions + .into_iter() + .map(|vers| Crate { + name: "foo".to_string(), + vers: vers.to_string(), + deps: vec![], + cksum: "0123456789asbcdef".to_string(), + features: Default::default(), + features2: None, + yanked: None, + links: None, + v: None, + }) + .collect::>(); + + let mut buffer = Vec::new(); + assert_ok!(write_crates(&crates, &mut buffer)); + assert_ok_eq!(String::from_utf8(buffer), "\ + {\"name\":\"foo\",\"vers\":\"0.1.0\",\"deps\":[],\"cksum\":\"0123456789asbcdef\",\"features\":{},\"yanked\":null}\n\ + {\"name\":\"foo\",\"vers\":\"1.0.0-beta.1\",\"deps\":[],\"cksum\":\"0123456789asbcdef\",\"features\":{},\"yanked\":null}\n\ + {\"name\":\"foo\",\"vers\":\"1.0.0\",\"deps\":[],\"cksum\":\"0123456789asbcdef\",\"features\":{},\"yanked\":null}\n\ + {\"name\":\"foo\",\"vers\":\"1.2.3\",\"deps\":[],\"cksum\":\"0123456789asbcdef\",\"features\":{},\"yanked\":null}\n\ + "); + } +} diff --git a/src/admin/delete_crate.rs b/src/admin/delete_crate.rs index 10571ae6c63..ad1b5407f29 100644 --- a/src/admin/delete_crate.rs +++ b/src/admin/delete_crate.rs @@ -1,3 +1,4 @@ +use crate::background_jobs::Job; use crate::{admin::dialoguer, config, db, models::Crate, schema::crates}; use diesel::prelude::*; @@ -54,5 +55,9 @@ fn delete(opts: Opts, conn: &mut PgConnection) { panic!("aborting transaction"); } - uploader.delete_index(&client, &krate.name).unwrap(); + if dotenv::var("FEATURE_INDEX_SYNC").is_ok() { + Job::enqueue_sync_to_index(&krate.name, conn).unwrap(); + } else { + uploader.delete_index(&client, &krate.name).unwrap(); + } } diff --git a/src/admin/delete_version.rs b/src/admin/delete_version.rs index 8c4f5f02510..b67842565b5 100644 --- a/src/admin/delete_version.rs +++ b/src/admin/delete_version.rs @@ -1,3 +1,4 @@ +use crate::background_jobs::Job; use crate::{ admin::dialoguer, db, @@ -57,4 +58,8 @@ fn delete(opts: Opts, conn: &mut PgConnection) { if !opts.yes && !dialoguer::confirm("commit?") { panic!("aborting transaction"); } + + if dotenv::var("FEATURE_INDEX_SYNC").is_ok() { + Job::enqueue_sync_to_index(&krate.name, conn).unwrap(); + } } diff --git a/src/admin/yank_version.rs b/src/admin/yank_version.rs index 7ec3c0115f1..ee8c108cba6 100644 --- a/src/admin/yank_version.rs +++ b/src/admin/yank_version.rs @@ -5,6 +5,7 @@ use crate::{ schema::versions, }; +use crate::background_jobs::Job; use diesel::prelude::*; #[derive(clap::Parser, Debug)] @@ -64,7 +65,11 @@ fn yank(opts: Opts, conn: &mut PgConnection) { .execute(conn) .unwrap(); - crate::worker::sync_yanked(krate.name, v.num) - .enqueue(conn) - .unwrap(); + if dotenv::var("FEATURE_INDEX_SYNC").is_ok() { + Job::enqueue_sync_to_index(&krate.name, conn).unwrap(); + } else { + crate::worker::sync_yanked(krate.name, v.num) + .enqueue(conn) + .unwrap(); + } } diff --git a/src/background_jobs.rs b/src/background_jobs.rs index 3e09ed14a10..f0470979f08 100644 --- a/src/background_jobs.rs +++ b/src/background_jobs.rs @@ -18,6 +18,8 @@ pub enum Job { IndexAddCrate(IndexAddCrateJob), IndexSquash, IndexSyncToHttp(IndexSyncToHttpJob), + SyncToGitIndex(SyncToIndexJob), + SyncToSparseIndex(SyncToIndexJob), IndexUpdateYanked(IndexUpdateYankedJob), NormalizeIndex(NormalizeIndexJob), RenderAndUploadReadme(RenderAndUploadReadmeJob), @@ -47,8 +49,47 @@ impl Job { const INDEX_UPDATE_YANKED: &str = "sync_yanked"; const NORMALIZE_INDEX: &str = "normalize_index"; const RENDER_AND_UPLOAD_README: &str = "render_and_upload_readme"; + const SYNC_TO_GIT_INDEX: &str = "sync_to_git_index"; + const SYNC_TO_SPARSE_INDEX: &str = "sync_to_sparse_index"; const UPDATE_DOWNLOADS: &str = "update_downloads"; + pub fn enqueue_sync_to_index( + krate: T, + conn: &mut PgConnection, + ) -> Result<(), EnqueueError> { + use crate::schema::background_jobs::dsl::*; + + let to_git = Self::sync_to_git_index(krate.to_string()); + let to_git = ( + job_type.eq(to_git.as_type_str()), + data.eq(to_git.to_value()?), + ); + + let to_sparse = Self::sync_to_sparse_index(krate.to_string()); + let to_sparse = ( + job_type.eq(to_sparse.as_type_str()), + data.eq(to_sparse.to_value()?), + ); + + diesel::insert_into(background_jobs) + .values(vec![to_git, to_sparse]) + .execute(conn)?; + + Ok(()) + } + + pub fn sync_to_git_index(krate: T) -> Job { + Job::SyncToGitIndex(SyncToIndexJob { + krate: krate.to_string(), + }) + } + + pub fn sync_to_sparse_index(krate: T) -> Job { + Job::SyncToSparseIndex(SyncToIndexJob { + krate: krate.to_string(), + }) + } + fn as_type_str(&self) -> &'static str { match self { Job::DailyDbMaintenance => Self::DAILY_DB_MAINTENANCE, @@ -59,6 +100,8 @@ impl Job { Job::IndexUpdateYanked(_) => Self::INDEX_UPDATE_YANKED, Job::NormalizeIndex(_) => Self::NORMALIZE_INDEX, Job::RenderAndUploadReadme(_) => Self::RENDER_AND_UPLOAD_README, + Job::SyncToGitIndex(_) => Self::SYNC_TO_GIT_INDEX, + Job::SyncToSparseIndex(_) => Self::SYNC_TO_SPARSE_INDEX, Job::UpdateDownloads => Self::UPDATE_DOWNLOADS, } } @@ -73,6 +116,8 @@ impl Job { Job::IndexUpdateYanked(inner) => serde_json::to_value(inner), Job::NormalizeIndex(inner) => serde_json::to_value(inner), Job::RenderAndUploadReadme(inner) => serde_json::to_value(inner), + Job::SyncToGitIndex(inner) => serde_json::to_value(inner), + Job::SyncToSparseIndex(inner) => serde_json::to_value(inner), Job::UpdateDownloads => Ok(serde_json::Value::Null), } } @@ -101,6 +146,8 @@ impl Job { Self::INDEX_UPDATE_YANKED => Job::IndexUpdateYanked(from_value(value)?), Self::NORMALIZE_INDEX => Job::NormalizeIndex(from_value(value)?), Self::RENDER_AND_UPLOAD_README => Job::RenderAndUploadReadme(from_value(value)?), + Self::SYNC_TO_GIT_INDEX => Job::SyncToGitIndex(from_value(value)?), + Self::SYNC_TO_SPARSE_INDEX => Job::SyncToSparseIndex(from_value(value)?), Self::UPDATE_DOWNLOADS => Job::UpdateDownloads, job_type => Err(PerformError::from(format!("Unknown job type {job_type}")))?, }) @@ -136,6 +183,8 @@ impl Job { args.base_url.as_deref(), args.pkg_path_in_vcs.as_deref(), ), + Job::SyncToGitIndex(args) => worker::sync_to_git_index(env, conn, &args.krate), + Job::SyncToSparseIndex(args) => worker::sync_to_sparse_index(env, conn, &args.krate), Job::UpdateDownloads => worker::perform_update_downloads(&mut *fresh_connection(pool)?), } } @@ -172,6 +221,11 @@ pub struct IndexSyncToHttpJob { pub(super) crate_name: String, } +#[derive(Serialize, Deserialize)] +pub struct SyncToIndexJob { + pub(super) krate: String, +} + #[derive(Serialize, Deserialize)] pub struct IndexUpdateYankedJob { pub(super) krate: String, diff --git a/src/config.rs b/src/config.rs index 736820cbb0c..e6583fd1586 100644 --- a/src/config.rs +++ b/src/config.rs @@ -47,6 +47,7 @@ pub struct Server { pub version_id_cache_ttl: Duration, pub cdn_user_agent: String, pub balance_capacity: BalanceCapacityConfig, + pub feature_index_sync: bool, } impl Default for Server { @@ -151,6 +152,7 @@ impl Default for Server { cdn_user_agent: dotenv::var("WEB_CDN_USER_AGENT") .unwrap_or_else(|_| "Amazon CloudFront".into()), balance_capacity: BalanceCapacityConfig::from_environment(), + feature_index_sync: dotenv::var("FEATURE_INDEX_SYNC").is_ok(), } } } diff --git a/src/controllers/krate/publish.rs b/src/controllers/krate/publish.rs index 9e0fe3b09d4..6cae3e46313 100644 --- a/src/controllers/krate/publish.rs +++ b/src/controllers/krate/publish.rs @@ -1,6 +1,7 @@ //! Functionality related to publishing a new crate or version of a crate. use crate::auth::AuthCheck; +use crate::background_jobs::Job; use axum::body::Bytes; use flate2::read::GzDecoder; use hex::ToHex; @@ -271,7 +272,12 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult for DependencyKind { + fn from(dk: IndexDependencyKind) -> Self { + match dk { + IndexDependencyKind::Normal => DependencyKind::Normal, + IndexDependencyKind::Build => DependencyKind::Build, + IndexDependencyKind::Dev => DependencyKind::Dev, + } + } +} + impl From for IndexDependencyKind { fn from(dk: DependencyKind) -> Self { match dk { diff --git a/src/models/krate.rs b/src/models/krate.rs index e4b1d540b60..73ca029163d 100644 --- a/src/models/krate.rs +++ b/src/models/krate.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use chrono::NaiveDateTime; use diesel::associations::Identifiable; use diesel::pg::Pg; @@ -9,7 +11,7 @@ use crate::app::App; use crate::controllers::helpers::pagination::*; use crate::models::version::TopVersions; use crate::models::{ - CrateOwner, CrateOwnerInvitation, NewCrateOwnerInvitationOutcome, Owner, OwnerKind, + CrateOwner, CrateOwnerInvitation, Dependency, NewCrateOwnerInvitationOutcome, Owner, OwnerKind, ReverseDependency, User, Version, }; use crate::util::errors::{cargo_err, AppResult}; @@ -432,6 +434,89 @@ impl Crate { Ok(rows.records_and_total()) } + + /// Gather all the necessary data to write an index metadata file + pub fn index_metadata( + &self, + conn: &mut PgConnection, + ) -> QueryResult> { + let mut versions: Vec = self.all_versions().load(conn)?; + + // We sort by `created_at` by default, but since tests run within a + // single database transaction the versions will all have the same + // `created_at` timestamp, so we sort by semver as a secondary key. + versions.sort_by_cached_key(|k| (k.created_at, semver::Version::parse(&k.num).ok())); + + let deps: Vec<(Dependency, String)> = Dependency::belonging_to(&versions) + .inner_join(crates::table) + .select((dependencies::all_columns, crates::name)) + .load(conn)?; + + let deps = deps.grouped_by(&versions); + + versions + .into_iter() + .zip(deps) + .map(|(version, deps)| { + let mut deps = deps + .into_iter() + .map(|(dep, name)| { + // If this dependency has an explicit name in `Cargo.toml` that + // means that the `name` we have listed is actually the package name + // that we're depending on. The `name` listed in the index is the + // Cargo.toml-written-name which is what cargo uses for + // `--extern foo=...` + let (name, package) = match dep.explicit_name { + Some(explicit_name) => (explicit_name, Some(name)), + None => (name, None), + }; + + cargo_registry_index::Dependency { + name, + req: dep.req, + features: dep.features, + optional: dep.optional, + default_features: dep.default_features, + kind: Some(dep.kind.into()), + package, + target: dep.target, + } + }) + .collect::>(); + + deps.sort(); + + let features: BTreeMap> = + serde_json::from_value(version.features).unwrap_or_default(); + let (features, features2): (BTreeMap<_, _>, BTreeMap<_, _>) = + features.into_iter().partition(|(_k, vals)| { + !vals + .iter() + .any(|v| v.starts_with("dep:") || v.contains("?/")) + }); + + let (features2, v) = if features2.is_empty() { + (None, None) + } else { + (Some(features2), Some(2)) + }; + + let krate = cargo_registry_index::Crate { + name: self.name.clone(), + vers: version.num.to_string(), + cksum: version.checksum, + yanked: Some(version.yanked), + deps, + features, + links: version.links, + features2, + v, + }; + + Ok(krate) + }) + .collect() + } } #[cfg(test)] diff --git a/src/tests/all.rs b/src/tests/all.rs index a1f905edefb..64dccc50931 100644 --- a/src/tests/all.rs +++ b/src/tests/all.rs @@ -34,6 +34,7 @@ mod dump_db; mod github_secret_scanning; mod krate; mod middleware; +mod models; mod not_found_error; mod owners; mod pagination; diff --git a/src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted.json b/src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted.json index 2dcdf9e930a..43208d50f2d 100644 --- a/src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted.json +++ b/src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted.json @@ -44,14 +44,14 @@ ], [ "content-length", - "154" + "309" ], [ "content-type", "text/plain" ] ], - "body": "{\"name\":\"foo_whitelist\",\"vers\":\"1.1.0\",\"deps\":[],\"cksum\":\"4e33dc59bbbc96645fc01945fb502507d1b7bd3a2d06227bf7b0fe8842f284c2\",\"features\":{},\"yanked\":false}\n" + "body": "{\"name\":\"foo_whitelist\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\" \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_whitelist\",\"vers\":\"1.1.0\",\"deps\":[],\"cksum\":\"4e33dc59bbbc96645fc01945fb502507d1b7bd3a2d06227bf7b0fe8842f284c2\",\"features\":{},\"yanked\":false}\n" }, "response": { "status": 200, diff --git a/src/tests/http-data/krate_publish_new_krate_twice.json b/src/tests/http-data/krate_publish_new_krate_twice.json index 85f687a0a32..17a120a671c 100644 --- a/src/tests/http-data/krate_publish_new_krate_twice.json +++ b/src/tests/http-data/krate_publish_new_krate_twice.json @@ -44,14 +44,14 @@ ], [ "content-length", - "150" + "301" ], [ "content-type", "text/plain" ] ], - "body": "{\"name\":\"foo_twice\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" + "body": "{\"name\":\"foo_twice\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\" \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_twice\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" }, "response": { "status": 200, diff --git a/src/tests/http-data/krate_publish_publish_after_removing_documentation.json b/src/tests/http-data/krate_publish_publish_after_removing_documentation.json index cf2bb2b771c..1abc9cebff3 100644 --- a/src/tests/http-data/krate_publish_publish_after_removing_documentation.json +++ b/src/tests/http-data/krate_publish_publish_after_removing_documentation.json @@ -44,14 +44,14 @@ ], [ "content-length", - "150" + "300" ], [ "content-type", "text/plain" ] ], - "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" + "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.0\",\"deps\":[],\"cksum\":\" \",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" }, "response": { "status": 200, @@ -104,14 +104,14 @@ ], [ "content-length", - "300" + "450" ], [ "content-type", "text/plain" ] ], - "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.2\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" + "body": "{\"name\":\"docscrate\",\"vers\":\"0.2.0\",\"deps\":[],\"cksum\":\" \",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.1\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n{\"name\":\"docscrate\",\"vers\":\"0.2.2\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" }, "response": { "status": 200, diff --git a/src/tests/http-data/team_publish_owned.json b/src/tests/http-data/team_publish_owned.json index f2edde531a9..aff042d6ec4 100644 --- a/src/tests/http-data/team_publish_owned.json +++ b/src/tests/http-data/team_publish_owned.json @@ -44,14 +44,14 @@ ], [ "content-length", - "155" + "311" ], [ "content-type", "text/plain" ] ], - "body": "{\"name\":\"foo_team_owned\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" + "body": "{\"name\":\"foo_team_owned\",\"vers\":\"0.99.0\",\"deps\":[],\"cksum\":\" \",\"features\":{},\"yanked\":false}\n{\"name\":\"foo_team_owned\",\"vers\":\"2.0.0\",\"deps\":[],\"cksum\":\"acb5604b126ac894c1eb11c4575bf2072fea61232a888e453770c79d7ed56419\",\"features\":{},\"yanked\":false}\n" }, "response": { "status": 200, diff --git a/src/tests/models/krate.rs b/src/tests/models/krate.rs new file mode 100644 index 00000000000..403f27e3344 --- /dev/null +++ b/src/tests/models/krate.rs @@ -0,0 +1,47 @@ +use crate::builders::{CrateBuilder, VersionBuilder}; +use crate::util::insta::assert_yaml_snapshot; +use crate::TestApp; +use chrono::{Days, Utc}; + +#[test] +fn index_metadata() { + let (app, _, user) = TestApp::init().with_user(); + let user = user.as_model(); + + app.db(|conn| { + let created_at_1 = Utc::now() + .checked_sub_days(Days::new(14)) + .unwrap() + .naive_utc(); + + let created_at_2 = Utc::now() + .checked_sub_days(Days::new(7)) + .unwrap() + .naive_utc(); + + let fooo = CrateBuilder::new("foo", user.id) + .version(VersionBuilder::new("0.1.0")) + .expect_build(conn); + + let metadata = fooo.index_metadata(conn).unwrap(); + assert_yaml_snapshot!(metadata); + + let bar = CrateBuilder::new("bar", user.id) + .version( + VersionBuilder::new("1.0.0-beta.1") + .created_at(created_at_1) + .yanked(true), + ) + .version(VersionBuilder::new("1.0.0").created_at(created_at_1)) + .version( + VersionBuilder::new("2.0.0") + .created_at(created_at_2) + .dependency(&fooo, None), + ) + .version(VersionBuilder::new("1.0.1").checksum("0123456789abcdef")) + .expect_build(conn); + + let metadata = bar.index_metadata(conn).unwrap(); + assert_yaml_snapshot!(metadata); + }); +} diff --git a/src/tests/models/mod.rs b/src/tests/models/mod.rs new file mode 100644 index 00000000000..c1ac1724be5 --- /dev/null +++ b/src/tests/models/mod.rs @@ -0,0 +1 @@ +mod krate; diff --git a/src/tests/models/snapshots/all__models__krate__index_metadata-2.snap b/src/tests/models/snapshots/all__models__krate__index_metadata-2.snap new file mode 100644 index 00000000000..2afe174ace3 --- /dev/null +++ b/src/tests/models/snapshots/all__models__krate__index_metadata-2.snap @@ -0,0 +1,36 @@ +--- +source: src/tests/models/krate.rs +expression: metadata +--- +- name: bar + vers: 1.0.0-beta.1 + deps: [] + cksum: " " + features: {} + yanked: true +- name: bar + vers: 1.0.0 + deps: [] + cksum: " " + features: {} + yanked: false +- name: bar + vers: 2.0.0 + deps: + - name: foo + req: ">= 0" + features: [] + optional: false + default_features: false + target: ~ + kind: normal + cksum: " " + features: {} + yanked: false +- name: bar + vers: 1.0.1 + deps: [] + cksum: "0123456789abcdef " + features: {} + yanked: false + diff --git a/src/tests/models/snapshots/all__models__krate__index_metadata.snap b/src/tests/models/snapshots/all__models__krate__index_metadata.snap new file mode 100644 index 00000000000..8822c445111 --- /dev/null +++ b/src/tests/models/snapshots/all__models__krate__index_metadata.snap @@ -0,0 +1,11 @@ +--- +source: src/tests/models/krate.rs +expression: metadata +--- +- name: foo + vers: 0.1.0 + deps: [] + cksum: " " + features: {} + yanked: false + diff --git a/src/tests/util/test_app.rs b/src/tests/util/test_app.rs index 4808c0940db..a9c389b0634 100644 --- a/src/tests/util/test_app.rs +++ b/src/tests/util/test_app.rs @@ -359,6 +359,7 @@ fn simple_config() -> config::Server { version_id_cache_ttl: Duration::from_secs(5 * 60), cdn_user_agent: "Amazon CloudFront".to_string(), balance_capacity: BalanceCapacityConfig::for_testing(), + feature_index_sync: true, } } diff --git a/src/worker/git.rs b/src/worker/git.rs index 3ff7ff21ba1..1520289bead 100644 --- a/src/worker/git.rs +++ b/src/worker/git.rs @@ -1,14 +1,15 @@ use crate::background_jobs::{ Environment, IndexAddCrateJob, IndexSyncToHttpJob, IndexUpdateYankedJob, Job, NormalizeIndexJob, }; +use crate::models; use crate::schema; use crate::swirl::PerformError; use anyhow::Context; use cargo_registry_index::{Crate, Repository}; use chrono::Utc; use diesel::prelude::*; -use std::fs::{self, OpenOptions}; -use std::io::{BufRead, BufReader, ErrorKind}; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufRead, BufReader, ErrorKind, Write}; use std::process::Command; #[instrument(skip_all, fields(krate.name = ?krate.name, krate.vers = ?krate.vers))] @@ -74,6 +75,98 @@ pub fn update_crate_index(crate_name: String) -> Job { Job::IndexSyncToHttp(IndexSyncToHttpJob { crate_name }) } +/// Regenerates or removes an index file for a single crate +#[instrument(skip_all, fields(krate.name = ?krate))] +pub fn sync_to_git_index( + env: &Environment, + conn: &mut PgConnection, + krate: &str, +) -> Result<(), PerformError> { + info!("Syncing to git index"); + + let new = get_index_data(krate, conn).context("Failed to get index data")?; + + let repo = env.lock_index()?; + let dst = repo.index_file(krate); + + // Read the previous crate contents + let old = match fs::read_to_string(&dst) { + Ok(content) => Some(content), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + + match (old, new) { + (None, Some(new)) => { + fs::create_dir_all(dst.parent().unwrap())?; + let mut file = File::create(&dst)?; + file.write_all(new.as_bytes())?; + repo.commit_and_push(&format!("Creating crate `{}`", krate), &dst)?; + } + (Some(old), Some(new)) if old != new => { + let mut file = File::create(&dst)?; + file.write_all(new.as_bytes())?; + repo.commit_and_push(&format!("Updating crate `{}`", krate), &dst)?; + } + (Some(_old), None) => { + fs::remove_file(&dst)?; + repo.commit_and_push(&format!("Deleting crate `{}`", krate), &dst)?; + } + _ => debug!("Skipping sync because index is up-to-date"), + } + + Ok(()) +} + +/// Regenerates or removes an index file for a single crate +#[instrument(skip_all, fields(krate.name = ?krate))] +pub fn sync_to_sparse_index( + env: &Environment, + conn: &mut PgConnection, + krate: &str, +) -> Result<(), PerformError> { + info!("Syncing to sparse index"); + + let content = get_index_data(krate, conn).context("Failed to get index data")?; + + env.uploader + .sync_index(env.http_client(), krate, content) + .context("Failed to sync index data")?; + + if let Some(cloudfront) = env.cloudfront() { + let path = Repository::relative_index_file_for_url(krate); + + info!(%path, "Invalidating index file on CloudFront"); + cloudfront + .invalidate(env.http_client(), &path) + .context("Failed to invalidate CloudFront")?; + } + + Ok(()) +} + +#[instrument(skip_all, fields(krate.name = ?name))] +pub fn get_index_data(name: &str, conn: &mut PgConnection) -> anyhow::Result> { + debug!("Looking up crate by name"); + let Some(krate): Option = models::Crate::by_exact_name(name).first(conn).optional()? else { + return Ok(None); + }; + + debug!("Gathering remaining index data"); + let crates = krate + .index_metadata(conn) + .context("Failed to gather index metadata")?; + + debug!("Serializing index data"); + let mut bytes = Vec::new(); + cargo_registry_index::write_crates(&crates, &mut bytes) + .context("Failed to serialize index metadata")?; + + let str = String::from_utf8(bytes).context("Failed to decode index metadata as utf8")?; + + Ok(Some(str)) +} + /// Yanks or unyanks a crate version. This requires finding the index /// file, deserlialise the crate from JSON, change the yank boolean to /// `true` or `false`, write all the lines back out, and commit and diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 51f6eb3f9a0..c5be80bc827 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -20,7 +20,7 @@ pub(crate) use daily_db_maintenance::perform_daily_db_maintenance; pub(crate) use dump_db::perform_dump_db; pub(crate) use git::{ perform_index_add_crate, perform_index_squash, perform_index_sync_to_http, - perform_index_update_yanked, perform_normalize_index, + perform_index_update_yanked, perform_normalize_index, sync_to_git_index, sync_to_sparse_index, }; pub(crate) use readmes::perform_render_and_upload_readme; pub(crate) use update_downloads::perform_update_downloads;