diff --git a/Cargo.lock b/Cargo.lock index 9aaecd10..425d307b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -122,7 +122,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -152,6 +152,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -464,7 +470,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -564,7 +570,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -911,6 +917,18 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -948,7 +966,7 @@ version = "9.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" dependencies = [ - "base64", + "base64 0.21.7", "js-sys", "pem", "ring", @@ -990,6 +1008,16 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.20" @@ -1106,7 +1134,7 @@ checksum = "9fc61913d67d5ed8e04b55c611ec7450ab811f83aa424f3bd00782c6b0296ae7" dependencies = [ "arc-swap", "async-trait", - "base64", + "base64 0.21.7", "bytes", "cfg-if", "chrono", @@ -1167,7 +1195,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -1198,6 +1226,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + [[package]] name = "password-hash" version = "0.4.2" @@ -1238,7 +1291,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64", + "base64 0.21.7", "serde", ] @@ -1276,7 +1329,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -1309,7 +1362,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef" dependencies = [ - "base64", + "base64 0.21.7", "indexmap", "line-wrap", "quick-xml", @@ -1323,11 +1376,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -1356,7 +1418,9 @@ dependencies = [ "pdb", "pep440_rs", "rayon", - "reqwest", + "reqwest 0.11.24", + "reqwest-middleware", + "reqwest-retry", "scroll 0.12.0", "semver", "serde", @@ -1390,11 +1454,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] [[package]] name = "rayon" @@ -1416,6 +1504,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1431,7 +1528,7 @@ version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -1454,7 +1551,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -1465,7 +1562,88 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.52.0", +] + +[[package]] +name = "reqwest-middleware" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" +dependencies = [ + "anyhow", + "async-trait", + "http 1.0.0", + "reqwest 0.12.5", + "serde", + "thiserror", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom", + "http 1.0.0", + "hyper 1.2.0", + "parking_lot", + "reqwest 0.12.5", + "reqwest-middleware", + "retry-policies", + "thiserror", + "tokio", + "tracing", + "wasm-timer", +] + +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand", ] [[package]] @@ -1547,7 +1725,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -1556,7 +1734,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b" dependencies = [ - "base64", + "base64 0.21.7", "rustls-pki-types", ] @@ -1619,6 +1797,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "scroll" version = "0.11.0" @@ -1642,7 +1826,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -1710,7 +1894,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -1888,9 +2072,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.50" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -1903,6 +2087,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "system-configuration" version = "0.5.1" @@ -1960,22 +2150,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -2143,7 +2333,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", ] [[package]] @@ -2300,7 +2490,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", "wasm-bindgen-shared", ] @@ -2334,7 +2524,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2358,6 +2548,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.68" @@ -2541,6 +2746,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xattr" version = "1.3.1" @@ -2561,6 +2776,27 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + [[package]] name = "zeroize" version = "1.7.0" diff --git a/Cargo.toml b/Cargo.toml index f8d5bf84..ba5b32ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ pdb = "0.8.0" pep440_rs = "0.6.6" rayon = "1.8.1" reqwest = { version = "0.11.24", features = ["rustls", "stream"] } +reqwest-middleware = "0.4.0" +reqwest-retry = "0.7.0" scroll = "0.12.0" semver = "1.0.22" serde = { version = "1.0.197", features = ["derive"] } diff --git a/src/github.rs b/src/github.rs index 5197e775..bfc03d01 100644 --- a/src/github.rs +++ b/src/github.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::str::FromStr; - use { crate::release::{ bootstrap_llvm, produce_install_only, produce_install_only_stripped, RELEASE_TRIPLES, @@ -18,16 +16,39 @@ use { Octocrab, OctocrabBuilder, }, rayon::prelude::*, + reqwest::StatusCode, + reqwest_middleware::{reqwest, ClientBuilder, ClientWithMiddleware}, + reqwest_retry::{ + default_on_request_failure, policies::ExponentialBackoff, RetryTransientMiddleware, + Retryable, RetryableStrategy, + }, sha2::{Digest, Sha256}, std::{ collections::{BTreeMap, BTreeSet, HashMap}, io::Read, path::PathBuf, + str::FromStr, }, url::Url, zip::ZipArchive, }; +/// A retry strategy for GitHub uploads. +struct GitHubUploadRetryStrategy; +impl RetryableStrategy for GitHubUploadRetryStrategy { + fn handle( + &self, + res: &std::result::Result, + ) -> Option { + match res { + // Retry on 403s, these indicate a transient failure. + Ok(success) if success.status() == StatusCode::FORBIDDEN => Some(Retryable::Transient), + Ok(_) => None, + Err(error) => default_on_request_failure(error), + } + } +} + async fn fetch_artifact( client: &Octocrab, org: &str, @@ -45,6 +66,7 @@ async fn fetch_artifact( } async fn upload_release_artifact( + client: &ClientWithMiddleware, auth_token: String, release: &Release, filename: String, @@ -74,8 +96,7 @@ async fn upload_release_artifact( // Octocrab doesn't yet support release artifact upload. And the low-level HTTP API // forces the use of strings on us. So we have to make our own HTTP client. - let response = reqwest::Client::builder() - .build()? + let response = client .put(url) .header("Authorization", format!("Bearer {auth_token}")) .header("Content-Length", data.len()) @@ -451,42 +472,54 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( let mut digests = BTreeMap::new(); - let mut fs = vec![]; - - for (source, dest) in wanted_filenames { - if !filenames.contains(&source) { - continue; - } - - let file_data = Bytes::copy_from_slice(&std::fs::read(dist_dir.join(&source))?); + let retry_policy = ExponentialBackoff::builder().build_with_max_retries(5); + let raw_client = ClientBuilder::new(reqwest_middleware::reqwest::Client::new()) + .with(RetryTransientMiddleware::new_with_policy_and_strategy( + retry_policy, + GitHubUploadRetryStrategy, + )) + .build(); - let mut digest = Sha256::new(); - digest.update(&file_data); + { + let mut fs = vec![]; - let digest = hex::encode(digest.finalize()); - - digests.insert(dest.clone(), digest.clone()); + for (source, dest) in wanted_filenames { + if !filenames.contains(&source) { + continue; + } - fs.push(upload_release_artifact( - token.clone(), - &release, - dest.clone(), - file_data, - dry_run, - )); - fs.push(upload_release_artifact( - token.clone(), - &release, - format!("{}.sha256", dest), - Bytes::copy_from_slice(format!("{}\n", digest).as_bytes()), - dry_run, - )); - } + let file_data = Bytes::copy_from_slice(&std::fs::read(dist_dir.join(&source))?); + + let mut digest = Sha256::new(); + digest.update(&file_data); + + let digest = hex::encode(digest.finalize()); + + digests.insert(dest.clone(), digest.clone()); + + fs.push(upload_release_artifact( + &raw_client, + token.clone(), + &release, + dest.clone(), + file_data, + dry_run, + )); + fs.push(upload_release_artifact( + &raw_client, + token.clone(), + &release, + format!("{}.sha256", dest), + Bytes::copy_from_slice(format!("{}\n", digest).as_bytes()), + dry_run, + )); + } - let mut buffered = futures::stream::iter(fs).buffer_unordered(16); + let mut buffered = futures::stream::iter(fs).buffer_unordered(16); - while let Some(res) = buffered.next().await { - res?; + while let Some(res) = buffered.next().await { + res?; + } } let shasums = digests @@ -498,6 +531,7 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( std::fs::write(dist_dir.join("SHA256SUMS"), shasums.as_bytes())?; upload_release_artifact( + &raw_client, token.clone(), &release, "SHA256SUMS".to_string(),