Merge pull request #8010 from Turbo87/cdn-logs

Implement CDN log file parsers

Author: Turbo87

Cargo.lock

@@ -0,0 +1,31 @@
+[package]
+name = "crates_io_cdn_logs"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+edition = "2021"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = "=1.0.79"
+async-compression = { version = "=0.4.6", features = ["gzip", "tokio", "zstd"] }
+chrono = { version = "=0.4.33", features = ["serde"] }
+percent-encoding = "=2.3.1"
+semver = "=1.0.21"
+serde = { version = "=1.0.196", features = ["derive"] }
+serde_json = "=1.0.113"
+tokio = { version = "=1.35.1", features = ["io-util"] }
+tracing = "=0.1.40"
+
+[dev-dependencies]
+claims = "=0.7.1"
+clap = { version = "=4.4.18", features = ["derive"] }
+criterion = { version = "=0.5.1", features = ["async_tokio"] }
+insta = "=1.34.0"
+tokio = { version = "=1.35.1", features = ["fs", "macros", "rt", "rt-multi-thread"] }
+tracing-subscriber = { version = "=0.3.18", features = ["env-filter"] }
+
+[[bench]]
+name = "count_downloads"
+harness = false

crates_io_cdn_logs/Cargo.toml

@@ -0,0 +1,6 @@
+crates_io_cdn_logs
+===============================================================================
+
+This package contains code to parse the log files from the crates.io CDNs
+(AWS CloudFront and Fastly) and to count how often crates/versions are
+downloaded each day.

crates_io_cdn_logs/README.md

@@ -0,0 +1,29 @@
+use crates_io_cdn_logs::{cloudfront, fastly};
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use std::io::Cursor;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let bytes = include_bytes!("../test_data/cloudfront/basic.log");
+    c.bench_function("cloudfront", |b| {
+        // Insert a call to `to_async` to convert the bencher to async mode.
+        // The timing loops are the same as with the normal bencher.
+        b.to_async(&rt)
+            .iter(|| cloudfront::count_downloads(black_box(Cursor::new(bytes))));
+    });
+
+    let bytes = include_bytes!("../test_data/fastly/basic.log");
+    c.bench_function("fastly", |b| {
+        // Insert a call to `to_async` to convert the bencher to async mode.
+        // The timing loops are the same as with the normal bencher.
+        b.to_async(&rt)
+            .iter(|| fastly::count_downloads(black_box(Cursor::new(bytes))));
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);