diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 53ae91d57..9c2471921 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -13,8 +13,8 @@ use crate::docbuilder::Limits; use crate::error::Result; use crate::repositories::RepositoryStatsUpdater; use crate::storage::{ - CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list, rustdoc_archive_path, - rustdoc_json_path, source_archive_path, + RustdocJsonFormatVersion, get_file_list, rustdoc_archive_path, rustdoc_json_path, + source_archive_path, }; use crate::utils::{ CargoMetadata, ConfigName, copy_dir_all, get_config, parse_rustc_version, report_error, @@ -909,21 +909,11 @@ impl RustwideBuilder { .context("couldn't parse rustdoc json to find format version")? }; - let compressed_json: Vec = { - let _span = - info_span!("compress_json", file_size = json_filename.metadata()?.len()).entered(); - - compress( - BufReader::new(File::open(&json_filename)?), - CompressionAlgorithm::Zstd, - )? - }; - for format_version in [format_version, RustdocJsonFormatVersion::Latest] { let _span = info_span!("store_json", %format_version).entered(); let path = rustdoc_json_path(name, version, target, format_version); - self.storage.store_one(&path, compressed_json.clone())?; + self.storage.store_path(&path, &json_filename)?; self.storage.set_public_access(&path, true)?; } @@ -1495,10 +1485,10 @@ mod tests { .collect(); json_files.sort(); assert!(json_files[0].starts_with(&format!("empty-library_1.0.0_{target}_"))); - assert!(json_files[0].ends_with(".json.zst")); + assert!(json_files[0].ends_with(".json")); assert_eq!( json_files[1], - format!("empty-library_1.0.0_{target}_latest.json.zst") + format!("empty-library_1.0.0_{target}_latest.json") ); if target == &default_target { diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 13a2b3c27..197266c56 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -24,7 +24,8 @@ use mime::Mime; use path_slash::PathExt; use serde_with::{DeserializeFromStr, SerializeDisplay}; use std::{ - fmt, fs, + fmt, + fs::{self, File}, io::{self, BufReader}, num::ParseIntError, ops::RangeInclusive, @@ -569,6 +570,33 @@ impl AsyncStorage { Ok(alg) } + #[instrument(skip(self))] + pub(crate) async fn store_path( + &self, + target_path: impl Into + std::fmt::Debug, + source_path: impl AsRef + std::fmt::Debug, + ) -> Result { + let target_path = target_path.into(); + let source_path = source_path.as_ref(); + + let alg = CompressionAlgorithm::default(); + let content = compress(BufReader::new(File::open(source_path)?), alg)?; + + let mime = detect_mime(&target_path).to_owned(); + + self.store_inner(vec![Blob { + path: target_path, + mime, + content, + compression: Some(alg), + // this field is ignored by the backend + date_updated: Utc::now(), + }]) + .await?; + + Ok(alg) + } + async fn store_inner(&self, batch: Vec) -> Result<()> { match &self.backend { StorageBackend::Database(db) => db.store_batch(batch).await, @@ -777,6 +805,18 @@ impl Storage { self.runtime.block_on(self.inner.store_one(path, content)) } + // Store file into the backend at the given path (also used to detect mime type), returns the + // chosen compression algorithm + #[instrument(skip(self))] + pub(crate) fn store_path( + &self, + target_path: impl Into + std::fmt::Debug, + source_path: impl AsRef + std::fmt::Debug, + ) -> Result { + self.runtime + .block_on(self.inner.store_path(target_path, source_path)) + } + /// sync wrapper for the list_prefix function /// purely for testing purposes since it collects all files into a Vec. #[cfg(test)] @@ -843,7 +883,7 @@ pub(crate) fn rustdoc_json_path( format_version: RustdocJsonFormatVersion, ) -> String { format!( - "rustdoc-json/{name}/{version}/{target}/{name}_{version}_{target}_{format_version}.json.zst" + "rustdoc-json/{name}/{version}/{target}/{name}_{version}_{target}_{format_version}.json" ) } diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 8c443e939..cd0189a10 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -3155,7 +3155,7 @@ mod test { web.assert_redirect_cached_unchecked( &format!("/crate/dummy/{request_path_suffix}"), &format!("https://static.docs.rs/rustdoc-json/dummy/{redirect_version}/{redirect_target}/\ - dummy_{redirect_version}_{redirect_target}_{redirect_format_version}.json.zst"), + dummy_{redirect_version}_{redirect_target}_{redirect_format_version}.json"), CachePolicy::ForeverInCdn, &env.config(), )