From 0d4b804171acd307bdac6eecd3b49bd8b2fb2968 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 14 Feb 2023 10:00:07 +0000 Subject: [PATCH 1/3] CI validates WASM support For now failure is allowed as no work was done, but this should confirm the crate can at least be compiled to that target. We try different targets, including WASI, for good measure, and already build crates that are naturally working. --- .github/workflows/wasm.yml | 35 +++++++++++++++++++++++++++++++++++ git-path/src/convert.rs | 12 ++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/wasm.yml diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 00000000000..34684919cd6 --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,35 @@ +name: WASM + +on: + push: + branches: [ main ] + tags-ignore: [ '*' ] + paths: + - '.github/**' + - 'git-pack/**' + - '*.toml' + pull_request: + branches: [ main ] + paths: + - '.github/**' + - 'git-pack/**' + - '*.toml' + +jobs: + wasm: + name: WebAssembly + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + target: [ wasm32-unknown-unknown, wasm32-wasi ] + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update stable && rustup default stable && rustup target add ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + - run: set +x; for name in git-actor git-attributes git-bitmap git-chunk git-command git-commitgraph git-date git-glob git-hash git-hashtable git-mailmap git-object git-packetline git-path git-pathspec git-quote git-refspec git-revision git-traverse git-validate; do (cd $name && cargo build --target ${{ matrix.target }}); done + name: crates without feature toggles + - run: set +x; for feature in progress fs-walkdir-parallel parallel io-pipe crc32 zlib zlib-rust-backend fast-sha1 rustsha1 cache-efficiency-debug; do (cd git-features && cargo build --features $feature --target ${{ matrix.target }}); done + name: features of git-features +# - run: cargo build -p git-pack --target ${{ matrix.target }} // TODO: make something like it work diff --git a/git-path/src/convert.rs b/git-path/src/convert.rs index f1be9d65244..6a949529f58 100644 --- a/git-path/src/convert.rs +++ b/git-path/src/convert.rs @@ -54,7 +54,7 @@ pub fn try_into_bstr<'a>(path: impl Into>) -> Result use std::os::wasi::ffi::OsStringExt; path.into_os_string().into_vec().into() }; - #[cfg(not(unix))] + #[cfg(not(any(unix, target_os = "wasi")))] let p: BString = path.into_os_string().into_string().map_err(|_| Utf8Error)?.into(); p }), @@ -69,7 +69,7 @@ pub fn try_into_bstr<'a>(path: impl Into>) -> Result use std::os::wasi::ffi::OsStrExt; path.as_os_str().as_bytes().into() }; - #[cfg(not(unix))] + #[cfg(not(any(unix, target_os = "wasi")))] let p: &BStr = path.to_str().ok_or(Utf8Error)?.as_bytes().into(); p }), @@ -94,11 +94,11 @@ pub fn try_from_byte_slice(input: &[u8]) -> Result<&Path, Utf8Error> { OsStr::from_bytes(input).as_ref() }; #[cfg(target_os = "wasi")] - let p = { + let p: &Path = { use std::os::wasi::ffi::OsStrExt; OsStr::from_bytes(input).as_ref() }; - #[cfg(not(unix))] + #[cfg(not(any(unix, target_os = "wasi")))] let p = Path::new(std::str::from_utf8(input).map_err(|_| Utf8Error)?); Ok(p) } @@ -126,11 +126,11 @@ pub fn try_from_bstring(input: impl Into) -> Result std::ffi::OsString::from_vec(input.into()).into() }; #[cfg(target_os = "wasi")] - let p = { + let p: PathBuf = { use std::os::wasi::ffi::OsStringExt; std::ffi::OsString::from_vec(input.into()).into() }; - #[cfg(not(unix))] + #[cfg(not(any(unix, target_os = "wasi")))] let p = { use bstr::ByteVec; PathBuf::from( From f0e40ecddaf1211f76ed60ef30cf03dcfd53a7ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 14 Feb 2023 10:21:56 +0000 Subject: [PATCH 2/3] feat: add `wasm` feature toggle to allow compilation to wasm32-unknown-unknown --- .github/workflows/wasm.yml | 2 ++ Cargo.lock | 3 +++ git-diff/Cargo.toml | 4 ++++ git-pack/Cargo.toml | 2 ++ git-pack/src/lib.rs | 2 ++ 5 files changed, 13 insertions(+) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 34684919cd6..54be43f3bbf 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -32,4 +32,6 @@ jobs: name: crates without feature toggles - run: set +x; for feature in progress fs-walkdir-parallel parallel io-pipe crc32 zlib zlib-rust-backend fast-sha1 rustsha1 cache-efficiency-debug; do (cd git-features && cargo build --features $feature --target ${{ matrix.target }}); done name: features of git-features + - run: set +x; for name in git-diff; do (cd $name && cargo build --features wasm --target ${{ matrix.target }}); done + name: crates with 'wasm' feature # - run: cargo build -p git-pack --target ${{ matrix.target }} // TODO: make something like it work diff --git a/Cargo.lock b/Cargo.lock index 9cfa6879045..a8908c434a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1181,8 +1181,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1407,6 +1409,7 @@ dependencies = [ name = "git-diff" version = "0.26.1" dependencies = [ + "getrandom", "git-hash 0.10.2", "git-object 0.26.1", "imara-diff", diff --git a/git-diff/Cargo.toml b/git-diff/Cargo.toml index 2c8edab2c77..7751fa04a61 100644 --- a/git-diff/Cargo.toml +++ b/git-diff/Cargo.toml @@ -11,7 +11,10 @@ rust-version = "1.64" autotests = false [features] +## Data structures implement `serde::Serialize` and `serde::Deserialize`. serde1 = ["serde", "git-hash/serde1", "git-object/serde1"] +## Make it possible to compile to the `wasm32-unknown-unknown` target. +wasm = ["dep:getrandom"] [lib] doctest = false @@ -22,3 +25,4 @@ git-object = { version = "^0.26.1", path = "../git-object" } thiserror = "1.0.32" imara-diff = "0.1.3" serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"]} +getrandom = { version = "0.2.8", optional = true, default-features = false, features = ["js"] } diff --git a/git-pack/Cargo.toml b/git-pack/Cargo.toml index f18499f481e..dd059836c3a 100644 --- a/git-pack/Cargo.toml +++ b/git-pack/Cargo.toml @@ -24,6 +24,8 @@ pack-cache-lru-dynamic = ["clru"] object-cache-dynamic = ["clru"] ## Data structures implement `serde::Serialize` and `serde::Deserialize`. serde1 = ["serde", "git-object/serde1"] +## Make it possible to compile to the `wasm32-unknown-unknown` target. +wasm = ["git-diff/wasm"] [dependencies] git-features = { version = "^0.26.4", path = "../git-features", features = ["crc32", "rustsha1", "progress", "zlib"] } diff --git a/git-pack/src/lib.rs b/git-pack/src/lib.rs index 65c0080ae4c..60a7c837c6f 100755 --- a/git-pack/src/lib.rs +++ b/git-pack/src/lib.rs @@ -17,6 +17,8 @@ )] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![deny(missing_docs, rust_2018_idioms, unsafe_code)] +#![cfg(all(target_arch = "wasm32", target_os = "unknown", not(feature = "wasm")))] +compile_error!("the wasm32-unknown-unknown target is not supported by default, and the \"wasm\" feature is required."); /// pub mod bundle; From 6c4c196c9bc6c2171dc4dc58b69bd5ef53226e29 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 15 Feb 2023 16:15:59 +0000 Subject: [PATCH 3/3] feat!: add `wasm` feature toggle to let parts of `git-pack` build on wasm32. It's a breaking change because we also start using the `dep:` syntax for declaring references to optional dependencies, which will prevent them from being automatically available as features. Besides that, it adds the `wasm` feature toggle to allow compiling to `wasm32` targets. --- .github/workflows/wasm.yml | 5 +++-- git-pack/Cargo.toml | 20 ++++++++++++-------- git-pack/src/bundle/mod.rs | 1 + git-pack/src/lib.rs | 7 ++----- git-pack/tests/Cargo.toml | 2 +- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 54be43f3bbf..b7e535b1da1 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -32,6 +32,7 @@ jobs: name: crates without feature toggles - run: set +x; for feature in progress fs-walkdir-parallel parallel io-pipe crc32 zlib zlib-rust-backend fast-sha1 rustsha1 cache-efficiency-debug; do (cd git-features && cargo build --features $feature --target ${{ matrix.target }}); done name: features of git-features - - run: set +x; for name in git-diff; do (cd $name && cargo build --features wasm --target ${{ matrix.target }}); done + - run: set +x; for name in git-diff git-pack; do (cd $name && cargo build --features wasm --target ${{ matrix.target }}); done name: crates with 'wasm' feature -# - run: cargo build -p git-pack --target ${{ matrix.target }} // TODO: make something like it work + - run: cd git-pack && cargo build --all-features --target ${{ matrix.target }} + name: git-pack with all features (including wasm) diff --git a/git-pack/Cargo.toml b/git-pack/Cargo.toml index dd059836c3a..71e6e349895 100644 --- a/git-pack/Cargo.toml +++ b/git-pack/Cargo.toml @@ -17,13 +17,13 @@ doctest = false ## Provide a fixed-size allocation-free LRU cache for packs. It's useful if caching is desired while keeping the memory footprint ## for the LRU-cache itself low. -pack-cache-lru-static = ["uluru"] +pack-cache-lru-static = ["dep:uluru"] ## Provide a hash-map based LRU cache whose eviction is based a memory cap calculated from object data. -pack-cache-lru-dynamic = ["clru"] +pack-cache-lru-dynamic = ["dep:clru"] ## If set, select algorithms may additionally use a full-object cache which is queried before the pack itself. -object-cache-dynamic = ["clru"] +object-cache-dynamic = ["dep:clru"] ## Data structures implement `serde::Serialize` and `serde::Deserialize`. -serde1 = ["serde", "git-object/serde1"] +serde1 = ["dep:serde", "git-object/serde1"] ## Make it possible to compile to the `wasm32-unknown-unknown` target. wasm = ["git-diff/wasm"] @@ -35,20 +35,24 @@ git-chunk = { version = "^0.4.1", path = "../git-chunk" } git-object = { version = "^0.26.1", path = "../git-object" } git-traverse = { version = "^0.22.1", path = "../git-traverse" } git-diff = { version = "^0.26.1", path = "../git-diff" } -git-tempfile = { version = "^3.0.0", path = "../git-tempfile" } git-hashtable = { version = "^0.1.1", path = "../git-hashtable" } -smallvec = "1.3.0" memmap2 = "0.5.0" -serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] } +smallvec = "1.3.0" bytesize = "1.0.1" parking_lot = { version = "0.12.0", default-features = false } thiserror = "1.0.26" uluru = { version = "3.0.0", optional = true } clru = { version = "0.6.1", optional = true } -dashmap = "5.1.0" +serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] } +## If enabled, `cargo doc` will see feature documentation from this manifest. document-features = { version = "0.2.0", optional = true } +dashmap = "5.1.0" + + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +git-tempfile = { version = "^3.0.0", path = "../git-tempfile" } [dev-dependencies] git-testtools = { path = "../tests/tools"} diff --git a/git-pack/src/bundle/mod.rs b/git-pack/src/bundle/mod.rs index 8889c95441a..ae187e1e42c 100644 --- a/git-pack/src/bundle/mod.rs +++ b/git-pack/src/bundle/mod.rs @@ -3,6 +3,7 @@ pub mod init; mod find; /// +#[cfg(not(feature = "wasm"))] pub mod write; /// diff --git a/git-pack/src/lib.rs b/git-pack/src/lib.rs index 60a7c837c6f..4578b048388 100755 --- a/git-pack/src/lib.rs +++ b/git-pack/src/lib.rs @@ -17,17 +17,15 @@ )] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![deny(missing_docs, rust_2018_idioms, unsafe_code)] -#![cfg(all(target_arch = "wasm32", target_os = "unknown", not(feature = "wasm")))] -compile_error!("the wasm32-unknown-unknown target is not supported by default, and the \"wasm\" feature is required."); /// pub mod bundle; /// A bundle of pack data and the corresponding pack index pub struct Bundle { /// The pack file corresponding to `index` - pub pack: crate::data::File, + pub pack: data::File, /// The index file corresponding to `pack` - pub index: crate::index::File, + pub index: index::File, } /// @@ -39,7 +37,6 @@ pub mod cache; pub mod data; mod find_traits; - pub use find_traits::{Find, FindExt}; /// diff --git a/git-pack/tests/Cargo.toml b/git-pack/tests/Cargo.toml index b3f0d4496df..aa40901a4d6 100644 --- a/git-pack/tests/Cargo.toml +++ b/git-pack/tests/Cargo.toml @@ -13,7 +13,7 @@ rust-version = "1.64" ## Provide a fixed-size allocation-free LRU cache for packs. It's useful if caching is desired while keeping the memory footprint ## for the LRU-cache itself low. -pack-cache-lru-static = ["git-pack/uluru"] +pack-cache-lru-static = ["git-pack/pack-cache-lru-static"] ## Provide a hash-map based LRU cache whose eviction is based a memory cap calculated from object data. pack-cache-lru-dynamic = ["git-pack/pack-cache-lru-dynamic"] ## If set, select algorithms may additionally use a full-object cache which is queried before the pack itself.