worker: Add NormalizeIndex background job

Author: Turbo87

src/background_jobs.rs

@@ -18,6 +18,7 @@ pub enum Job {
     IndexSquash(IndexSquashJob),
     IndexSyncToHttp(IndexSyncToHttpJob),
     IndexUpdateYanked(IndexUpdateYankedJob),
+    NormalizeIndex(NormalizeIndexJob),
     RenderAndUploadReadme(RenderAndUploadReadmeJob),
     UpdateDownloads(UpdateDownloadsJob),
 }
@@ -29,6 +30,7 @@ impl Job {
     const INDEX_SQUASH: &str = "squash_index";
     const INDEX_SYNC_TO_HTTP: &str = "update_crate_index";
     const INDEX_UPDATE_YANKED: &str = "sync_yanked";
+    const NORMALIZE_INDEX: &str = "normalize_index";
     const RENDER_AND_UPLOAD_README: &str = "render_and_upload_readme";
     const UPDATE_DOWNLOADS: &str = "update_downloads";
 
@@ -40,6 +42,7 @@ impl Job {
             Job::IndexSquash(_) => Self::INDEX_SQUASH,
             Job::IndexSyncToHttp(_) => Self::INDEX_SYNC_TO_HTTP,
             Job::IndexUpdateYanked(_) => Self::INDEX_UPDATE_YANKED,
+            Job::NormalizeIndex(_) => Self::NORMALIZE_INDEX,
             Job::RenderAndUploadReadme(_) => Self::RENDER_AND_UPLOAD_README,
             Job::UpdateDownloads(_) => Self::UPDATE_DOWNLOADS,
         }
@@ -53,6 +56,7 @@ impl Job {
             Job::IndexSquash(inner) => serde_json::to_value(inner),
             Job::IndexSyncToHttp(inner) => serde_json::to_value(inner),
             Job::IndexUpdateYanked(inner) => serde_json::to_value(inner),
+            Job::NormalizeIndex(inner) => serde_json::to_value(inner),
             Job::RenderAndUploadReadme(inner) => serde_json::to_value(inner),
             Job::UpdateDownloads(inner) => serde_json::to_value(inner),
         }
@@ -80,6 +84,7 @@ impl Job {
             Self::INDEX_SQUASH => Job::IndexSquash(from_value(value)?),
             Self::INDEX_SYNC_TO_HTTP => Job::IndexSyncToHttp(from_value(value)?),
             Self::INDEX_UPDATE_YANKED => Job::IndexUpdateYanked(from_value(value)?),
+            Self::NORMALIZE_INDEX => Job::NormalizeIndex(from_value(value)?),
             Self::RENDER_AND_UPLOAD_README => Job::RenderAndUploadReadme(from_value(value)?),
             Self::UPDATE_DOWNLOADS => Job::UpdateDownloads(from_value(value)?),
             job_type => Err(PerformError::from(format!("Unknown job type {job_type}")))?,
@@ -106,6 +111,7 @@ impl Job {
             Job::IndexUpdateYanked(args) => conn.with_connection(&|conn| {
                 worker::perform_index_update_yanked(env, conn, &args.krate, &args.version_num)
             }),
+            Job::NormalizeIndex(args) => worker::perform_normalize_index(env, args),
             Job::RenderAndUploadReadme(args) => conn.with_connection(&|conn| {
                 worker::perform_render_and_upload_readme(
                     conn,
@@ -150,6 +156,11 @@ pub struct IndexUpdateYankedJob {
     pub(super) version_num: String,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct NormalizeIndexJob {
+    pub dry_run: bool,
+}
+
 #[derive(Serialize, Deserialize)]
 pub struct RenderAndUploadReadmeJob {
     pub(super) version_id: i32,

src/worker/git.rs

@@ -1,5 +1,6 @@
 use crate::background_jobs::{
     Environment, IndexAddCrateJob, IndexSquashJob, IndexSyncToHttpJob, IndexUpdateYankedJob, Job,
+    NormalizeIndexJob,
 };
 use crate::schema;
 use crate::swirl::PerformError;
@@ -8,7 +9,7 @@ use cargo_registry_index::{Crate, Repository};
 use chrono::Utc;
 use diesel::prelude::*;
 use std::fs::{self, OpenOptions};
-use std::io::ErrorKind;
+use std::io::{BufRead, BufReader, ErrorKind};
 use std::process::Command;
 
 #[instrument(skip_all, fields(krate.name = ?krate.name, krate.vers = ?krate.vers))]
@@ -178,3 +179,77 @@ pub fn perform_index_squash(env: &Environment) -> Result<(), PerformError> {
 pub fn squash_index() -> Job {
     Job::IndexSquash(IndexSquashJob {})
 }
+
+pub fn perform_normalize_index(
+    env: &Environment,
+    args: NormalizeIndexJob,
+) -> Result<(), PerformError> {
+    info!("Normalizing the index");
+
+    let repo = env.lock_index()?;
+
+    let files = repo.get_files_modified_since(None)?;
+    let num_files = files.len();
+
+    for (i, file) in files.iter().enumerate() {
+        if i % 50 == 0 {
+            info!(num_files, i, ?file);
+        }
+
+        let crate_name = file.file_name().unwrap().to_str().unwrap();
+        let path = repo.index_file(crate_name);
+        if !path.exists() {
+            continue;
+        }
+
+        let mut body: Vec<u8> = Vec::new();
+        let file = fs::File::open(&path)?;
+        let reader = BufReader::new(file);
+        let mut versions = Vec::new();
+        for line in reader.lines() {
+            let line = line?;
+            if line.is_empty() {
+                continue;
+            }
+
+            let mut krate: Crate = serde_json::from_str(&line)?;
+            for dep in &mut krate.deps {
+                // Remove deps with empty features
+                dep.features.retain(|d| !d.is_empty());
+                // Set null DependencyKind to Normal
+                dep.kind = Some(
+                    dep.kind
+                        .unwrap_or(cargo_registry_index::DependencyKind::Normal),
+                );
+            }
+            krate.deps.sort();
+            versions.push(krate);
+        }
+        for version in versions {
+            serde_json::to_writer(&mut body, &version).unwrap();
+            body.push(b'\n');
+        }
+        fs::write(path, body)?;
+    }
+
+    info!("Committing normalization");
+    let msg = "Normalize index format\n\n\
+        More information can be found at https://github.com/rust-lang/crates.io/pull/5066";
+    repo.run_command(Command::new("git").args(["commit", "-am", msg]))?;
+
+    let branch = match args.dry_run {
+        false => "master",
+        true => "normalization-dry-run",
+    };
+
+    info!(?branch, "Pushing to upstream repository");
+    repo.run_command(Command::new("git").args(["push", "origin", &format!("HEAD:{branch}")]))?;
+
+    info!("Index normalization completed");
+
+    Ok(())
+}
+
+pub fn normalize_index(dry_run: bool) -> Job {
+    Job::NormalizeIndex(NormalizeIndexJob { dry_run })
+}

src/worker/mod.rs

@@ -12,15 +12,15 @@ mod update_downloads;
 
 pub use daily_db_maintenance::daily_db_maintenance;
 pub use dump_db::dump_db;
-pub use git::{add_crate, squash_index, sync_yanked};
+pub use git::{add_crate, normalize_index, squash_index, sync_yanked};
 pub use readmes::render_and_upload_readme;
 pub use update_downloads::update_downloads;
 
 pub(crate) use daily_db_maintenance::perform_daily_db_maintenance;
 pub(crate) use dump_db::perform_dump_db;
 pub(crate) use git::{
     perform_index_add_crate, perform_index_squash, perform_index_sync_to_http,
-    perform_index_update_yanked,
+    perform_index_update_yanked, perform_normalize_index,
 };
 pub(crate) use readmes::perform_render_and_upload_readme;
 pub(crate) use update_downloads::perform_update_downloads;