Generate crate index files from the database

Uses the newly added columns and index_metadata function
to re-generate the entire crate index file from the database
rather than a single line when a crate is published or yanked.

Test data changes are due to the test framework inserting crates
into the database, but not the index.

Author: arlosi

src/tests/builders/version.rs

@@ -43,7 +43,8 @@ impl<'a> VersionBuilder<'a> {
             num,
             size: 0,
             yanked: false,
-            checksum: String::new(),
+            checksum: "0000000000000000000000000000000000000000000000000000000000000000"
+                .to_string(),
             links: None,
         }
     }

src/tests/http-data/krate_publish_new_krate_too_big_but_whitelisted

@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "167"
+          "335"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "eyJuYW1lIjoiZm9vX3doaXRlbGlzdCIsInZlcnMiOiIxLjEuMCIsImRlcHMiOltdLCJja3N1bSI6IjRlMzNkYzU5YmJiYzk2NjQ1ZmMwMTk0NWZiNTAyNTA3ZDFiN2JkM2EyZDA2MjI3YmY3YjBmZTg4NDJmMjg0YzIiLCJmZWF0dXJlcyI6e30sInlhbmtlZCI6ZmFsc2UsImxpbmtzIjpudWxsfQo="
+      "body": "eyJuYW1lIjoiZm9vX3doaXRlbGlzdCIsInZlcnMiOiIwLjk5LjAiLCJkZXBzIjpbXSwiY2tzdW0iOiIwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwIiwiZmVhdHVyZXMiOnt9LCJ5YW5rZWQiOmZhbHNlLCJsaW5rcyI6bnVsbH0KeyJuYW1lIjoiZm9vX3doaXRlbGlzdCIsInZlcnMiOiIxLjEuMCIsImRlcHMiOltdLCJja3N1bSI6IjRlMzNkYzU5YmJiYzk2NjQ1ZmMwMTk0NWZiNTAyNTA3ZDFiN2JkM2EyZDA2MjI3YmY3YjBmZTg4NDJmMjg0YzIiLCJmZWF0dXJlcyI6e30sInlhbmtlZCI6ZmFsc2UsImxpbmtzIjpudWxsfQo="
     },
     "response": {
       "status": 200,

src/tests/http-data/krate_publish_new_krate_twice

@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "163"
+          "327"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "eyJuYW1lIjoiZm9vX3R3aWNlIiwidmVycyI6IjIuMC4wIiwiZGVwcyI6W10sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9Cg=="
+      "body": "eyJuYW1lIjoiZm9vX3R3aWNlIiwidmVycyI6IjAuOTkuMCIsImRlcHMiOltdLCJja3N1bSI6IjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLCJmZWF0dXJlcyI6e30sInlhbmtlZCI6ZmFsc2UsImxpbmtzIjpudWxsfQp7Im5hbWUiOiJmb29fdHdpY2UiLCJ2ZXJzIjoiMi4wLjAiLCJkZXBzIjpbXSwiY2tzdW0iOiJhY2I1NjA0YjEyNmFjODk0YzFlYjExYzQ1NzViZjIwNzJmZWE2MTIzMmE4ODhlNDUzNzcwYzc5ZDdlZDU2NDE5IiwiZmVhdHVyZXMiOnt9LCJ5YW5rZWQiOmZhbHNlLCJsaW5rcyI6bnVsbH0K"
     },
     "response": {
       "status": 200,

src/tests/http-data/krate_publish_publish_after_removing_documentation

@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "163"
+          "326"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "eyJuYW1lIjoiZG9jc2NyYXRlIiwidmVycyI6IjAuMi4xIiwiZGVwcyI6W10sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9Cg=="
+      "body": "eyJuYW1lIjoiZG9jc2NyYXRlIiwidmVycyI6IjAuMi4wIiwiZGVwcyI6W10sImNrc3VtIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9CnsibmFtZSI6ImRvY3NjcmF0ZSIsInZlcnMiOiIwLjIuMSIsImRlcHMiOltdLCJja3N1bSI6ImFjYjU2MDRiMTI2YWM4OTRjMWViMTFjNDU3NWJmMjA3MmZlYTYxMjMyYTg4OGU0NTM3NzBjNzlkN2VkNTY0MTkiLCJmZWF0dXJlcyI6e30sInlhbmtlZCI6ZmFsc2UsImxpbmtzIjpudWxsfQo="
     },
     "response": {
       "status": 200,
@@ -104,14 +104,14 @@
         ],
         [
           "content-length",
-          "326"
+          "489"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "eyJuYW1lIjoiZG9jc2NyYXRlIiwidmVycyI6IjAuMi4xIiwiZGVwcyI6W10sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9CnsibmFtZSI6ImRvY3NjcmF0ZSIsInZlcnMiOiIwLjIuMiIsImRlcHMiOltdLCJja3N1bSI6ImFjYjU2MDRiMTI2YWM4OTRjMWViMTFjNDU3NWJmMjA3MmZlYTYxMjMyYTg4OGU0NTM3NzBjNzlkN2VkNTY0MTkiLCJmZWF0dXJlcyI6e30sInlhbmtlZCI6ZmFsc2UsImxpbmtzIjpudWxsfQo="
+      "body": "eyJuYW1lIjoiZG9jc2NyYXRlIiwidmVycyI6IjAuMi4wIiwiZGVwcyI6W10sImNrc3VtIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9CnsibmFtZSI6ImRvY3NjcmF0ZSIsInZlcnMiOiIwLjIuMSIsImRlcHMiOltdLCJja3N1bSI6ImFjYjU2MDRiMTI2YWM4OTRjMWViMTFjNDU3NWJmMjA3MmZlYTYxMjMyYTg4OGU0NTM3NzBjNzlkN2VkNTY0MTkiLCJmZWF0dXJlcyI6e30sInlhbmtlZCI6ZmFsc2UsImxpbmtzIjpudWxsfQp7Im5hbWUiOiJkb2NzY3JhdGUiLCJ2ZXJzIjoiMC4yLjIiLCJkZXBzIjpbXSwiY2tzdW0iOiJhY2I1NjA0YjEyNmFjODk0YzFlYjExYzQ1NzViZjIwNzJmZWE2MTIzMmE4ODhlNDUzNzcwYzc5ZDdlZDU2NDE5IiwiZmVhdHVyZXMiOnt9LCJ5YW5rZWQiOmZhbHNlLCJsaW5rcyI6bnVsbH0K"
     },
     "response": {
       "status": 200,

src/tests/http-data/team_publish_owned

@@ -44,14 +44,14 @@
         ],
         [
           "content-length",
-          "168"
+          "337"
         ],
         [
           "content-type",
           "text/plain"
         ]
       ],
-      "body": "eyJuYW1lIjoiZm9vX3RlYW1fb3duZWQiLCJ2ZXJzIjoiMi4wLjAiLCJkZXBzIjpbXSwiY2tzdW0iOiJhY2I1NjA0YjEyNmFjODk0YzFlYjExYzQ1NzViZjIwNzJmZWE2MTIzMmE4ODhlNDUzNzcwYzc5ZDdlZDU2NDE5IiwiZmVhdHVyZXMiOnt9LCJ5YW5rZWQiOmZhbHNlLCJsaW5rcyI6bnVsbH0K"
+      "body": "eyJuYW1lIjoiZm9vX3RlYW1fb3duZWQiLCJ2ZXJzIjoiMC45OS4wIiwiZGVwcyI6W10sImNrc3VtIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9CnsibmFtZSI6ImZvb190ZWFtX293bmVkIiwidmVycyI6IjIuMC4wIiwiZGVwcyI6W10sImNrc3VtIjoiYWNiNTYwNGIxMjZhYzg5NGMxZWIxMWM0NTc1YmYyMDcyZmVhNjEyMzJhODg4ZTQ1Mzc3MGM3OWQ3ZWQ1NjQxOSIsImZlYXR1cmVzIjp7fSwieWFua2VkIjpmYWxzZSwibGlua3MiOm51bGx9Cg=="
     },
     "response": {
       "status": 200,

src/worker/git.rs

@@ -1,33 +1,18 @@
 use crate::background_jobs::Environment;
-use crate::schema;
+use crate::{models, schema};
 use anyhow::Context;
 use cargo_registry_index::Crate;
 use chrono::Utc;
 use diesel::prelude::*;
-use std::fs::{self, OpenOptions};
+use std::fs::{self, File};
 use std::io::ErrorKind;
 use std::process::Command;
-use swirl::PerformError;
+use swirl::{Job, PerformError};
 
 #[swirl::background_job]
 pub fn add_crate(env: &Environment, conn: &PgConnection, krate: Crate) -> Result<(), PerformError> {
-    use std::io::prelude::*;
-
-    let repo = env.lock_index()?;
-    let dst = repo.index_file(&krate.name);
-
-    // Add the crate to its relevant file
-    fs::create_dir_all(dst.parent().unwrap())?;
-    let mut file = OpenOptions::new().append(true).create(true).open(&dst)?;
-    serde_json::to_writer(&mut file, &krate)?;
-    file.write_all(b"\n")?;
-
     let message: String = format!("Updating crate `{}#{}`", krate.name, krate.vers);
-    repo.commit_and_push(&message, &dst)?;
-
-    // Queue another background job to update the http-based index as well.
-    update_crate_index(krate.name.clone()).enqueue(conn)?;
-    Ok(())
+    sync(env, conn, &krate.name, &message)
 }
 
 #[swirl::background_job]
@@ -47,10 +32,7 @@ pub fn update_crate_index(env: &Environment, crate_name: String) -> Result<(), P
     Ok(())
 }
 
-/// Yanks or unyanks a crate version. This requires finding the index
-/// file, deserlialise the crate from JSON, change the yank boolean to
-/// `true` or `false`, write all the lines back out, and commit and
-/// push the changes.
+/// Yanks or unyanks a crate version.
 #[swirl::background_job]
 pub fn sync_yanked(
     env: &Environment,
@@ -70,38 +52,39 @@ pub fn sync_yanked(
 
     trace!(?krate, ?version_num, yanked);
 
+    let action = if yanked { "Yanking" } else { "Unyanking" };
+    let message = format!("{action} crate `{krate}#{version_num}`");
+    sync(env, conn, &krate, &message)
+}
+
+/// Re-generates an index file from the database for a single crate.
+pub fn sync(
+    env: &Environment,
+    conn: &PgConnection,
+    krate: &str,
+    commit_message: &str,
+) -> Result<(), PerformError> {
+    use std::io::Write;
+    trace!(?krate, "Update crate from database");
+    let db_krate: models::Crate = models::Crate::by_exact_name(krate)
+        .first(conn)
+        .context("failed to find crate in database")?;
+    let new = db_krate
+        .index_metadata(conn)
+        .context("failed to generate index metadata")?;
+
     let repo = env.lock_index()?;
-    let dst = repo.index_file(&krate);
-
-    let prev = fs::read_to_string(&dst)?;
-    let new = prev
-        .lines()
-        .map(|line| {
-            let mut git_crate = serde_json::from_str::<Crate>(line)
-                .map_err(|_| format!("couldn't decode: `{line}`"))?;
-            if git_crate.name != krate || git_crate.vers != version_num {
-                return Ok(line.to_string());
-            }
-            git_crate.yanked = Some(yanked);
-            Ok(serde_json::to_string(&git_crate)?)
-        })
-        .collect::<Result<Vec<_>, PerformError>>();
-    let new = new?.join("\n") + "\n";
-
-    if new != prev {
-        fs::write(&dst, new.as_bytes())?;
-
-        let action = if yanked { "Yanking" } else { "Unyanking" };
-        let message = format!("{action} crate `{krate}#{version_num}`");
-
-        repo.commit_and_push(&message, &dst)?;
-    } else {
-        debug!("Skipping `yanked` update because index is up-to-date");
+    let dst = repo.index_file(krate);
+    if fs::read(&dst).ok().as_ref() == Some(&new) {
+        debug!("Skipping `{krate}` update because index is up-to-date");
+        return Ok(());
     }
+    fs::create_dir_all(dst.parent().unwrap())?;
+    File::create(&dst)?.write_all(&new)?;
+    repo.commit_and_push(commit_message, &dst)?;
 
     // Queue another background job to update the http-based index as well.
-    update_crate_index(krate.clone()).enqueue(conn)?;
-
+    update_crate_index(krate.to_string()).enqueue(conn)?;
     Ok(())
 }