From cf620f83e0e1dc720cdf129f49a80c97f831c974 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 4 Mar 2023 18:31:12 +0100 Subject: [PATCH 1/4] Inform about stale information about replacement objects --- SHORTCOMINGS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SHORTCOMINGS.md b/SHORTCOMINGS.md index cb37715af94..19b5d83edd0 100644 --- a/SHORTCOMINGS.md +++ b/SHORTCOMINGS.md @@ -25,6 +25,10 @@ This file is for tracking features that are less well implemented or less powerf * **Objects larger than 32 bits cannot be loaded on 32 bit systems** * in-memory representations objects cannot handle objects greater than the amount of addressable memory. * This will not affect git LFS though. + +### `gix` + +* object replacements are read once upon opening the repository from their refs and changes to these won't be picked up. ### `gix-url` From 3e69535630714205904fe64f511da28a3f2d7fb6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 5 Mar 2023 10:11:40 +0100 Subject: [PATCH 2/4] feat: `Repository::is_shallow()` to test if a repository is shallow. --- gix/src/config/cache/init.rs | 9 ++++++ gix/src/config/tree/sections/gitoxide.rs | 32 ++++++++++++++++--- gix/src/repository/location.rs | 16 ++++++++++ gix/src/repository/state.rs | 7 ++++ .../make_shallow_repo.tar.xz | 3 ++ gix/tests/fixtures/make_shallow_repo.sh | 28 ++++++++++++++++ gix/tests/repository/open.rs | 15 ++++++++- gix/tests/repository/state.rs | 22 +++++++++++++ 8 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 gix/tests/fixtures/generated-archives/make_shallow_repo.tar.xz create mode 100644 gix/tests/fixtures/make_shallow_repo.sh diff --git a/gix/src/config/cache/init.rs b/gix/src/config/cache/init.rs index dc76f78bbac..2aa4e4192cb 100644 --- a/gix/src/config/cache/init.rs +++ b/gix/src/config/cache/init.rs @@ -347,6 +347,15 @@ fn apply_environment_overrides( }, ], ), + ( + "gitoxide", + Some(Cow::Borrowed("core".into())), + git_prefix, + &[{ + let key = &gitoxide::Core::SHALLOW_FILE; + (env(key), key.name) + }], + ), ( "gitoxide", Some(Cow::Borrowed("author".into())), diff --git a/gix/src/config/tree/sections/gitoxide.rs b/gix/src/config/tree/sections/gitoxide.rs index 8c3defd0b68..1622139dedc 100644 --- a/gix/src/config/tree/sections/gitoxide.rs +++ b/gix/src/config/tree/sections/gitoxide.rs @@ -1,3 +1,4 @@ +use crate::config; use crate::config::tree::{keys, Gitoxide, Key, Section}; impl Gitoxide { @@ -5,6 +6,8 @@ impl Gitoxide { pub const ALLOW: Allow = Allow; /// The `gitoxide.author` section. pub const AUTHOR: Author = Author; + /// The `gitoxide.core` section. + pub const CORE: Core = Core; /// The `gitoxide.commit` section. pub const COMMIT: Commit = Commit; /// The `gitoxide.committer` section. @@ -39,6 +42,7 @@ impl Section for Gitoxide { &[ &Self::ALLOW, &Self::AUTHOR, + &Self::CORE, &Self::COMMIT, &Self::COMMITTER, &Self::HTTP, @@ -56,6 +60,29 @@ mod subsections { Tree, }; + /// The `Core` sub-section. + #[derive(Copy, Clone, Default)] + pub struct Core; + + impl Core { + /// The `gitoxide.core.shallowFile` key. + pub const SHALLOW_FILE: keys::Path = keys::Path::new_path("shallowFile", &Gitoxide::CORE) + .with_environment_override("GIT_SHALLOW_FILE") + .with_deviation( + "relative file paths will always be made relative to the git-common-dir, whereas `git` keeps them as is.", + ); + } + + impl Section for Core { + fn name(&self) -> &str { + "core" + } + + fn keys(&self) -> &[&dyn Key] { + &[&Self::SHALLOW_FILE] + } + } + /// The `Http` sub-section. #[derive(Copy, Clone, Default)] pub struct Http; @@ -341,6 +368,7 @@ mod subsections { } } } +pub use subsections::{Allow, Author, Commit, Committer, Core, Http, Https, Objects, Ssh, User}; pub mod validate { use std::error::Error; @@ -357,7 +385,3 @@ pub mod validate { } } } - -pub use subsections::{Allow, Author, Commit, Committer, Http, Https, Objects, Ssh, User}; - -use crate::config; diff --git a/gix/src/repository/location.rs b/gix/src/repository/location.rs index 0bb8ea2535d..f64641f80d8 100644 --- a/gix/src/repository/location.rs +++ b/gix/src/repository/location.rs @@ -1,5 +1,7 @@ +use std::borrow::Cow; use std::path::PathBuf; +use crate::config::tree::{gitoxide, Key}; use gix_path::realpath::MAX_SYMLINKS; impl crate::Repository { @@ -41,6 +43,20 @@ impl crate::Repository { crate::path::install_dir() } + /// Return the path to the `shallow` file which contains hashes, one per line, that describe commits that don't have their + /// parents within this repository. + pub fn shallow_file(&self) -> PathBuf { + let shallow_name = self + .config + .resolved + .string_filter_by_key( + gitoxide::Core::SHALLOW_FILE.logical_name().as_str(), + &mut self.filter_config_section(), + ) + .unwrap_or(Cow::Borrowed("shallow".into())); + self.common_dir().join(gix_path::from_bstr(shallow_name)) + } + /// Returns the relative path which is the components between the working tree and the current working dir (CWD). /// Note that there may be `None` if there is no work tree, even though the `PathBuf` will be empty /// if the CWD is at the root of the work tree. diff --git a/gix/src/repository/state.rs b/gix/src/repository/state.rs index 4034fe349db..c8619ca32c0 100644 --- a/gix/src/repository/state.rs +++ b/gix/src/repository/state.rs @@ -41,4 +41,11 @@ impl crate::Repository { None } } + + /// Return `true` if the repository is a shallow clone, i.e. contains history only up to a certain depth. + pub fn is_shallow(&self) -> bool { + self.shallow_file() + .metadata() + .map_or(false, |m| m.is_file() && m.len() > 0) + } } diff --git a/gix/tests/fixtures/generated-archives/make_shallow_repo.tar.xz b/gix/tests/fixtures/generated-archives/make_shallow_repo.tar.xz new file mode 100644 index 00000000000..d2da451c8b1 --- /dev/null +++ b/gix/tests/fixtures/generated-archives/make_shallow_repo.tar.xz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c56c269562ef67b1f8bd46640e6ad9d196cbc9c7c609300ffd6a8da3bc501852 +size 12632 diff --git a/gix/tests/fixtures/make_shallow_repo.sh b/gix/tests/fixtures/make_shallow_repo.sh new file mode 100644 index 00000000000..236d5e00d1c --- /dev/null +++ b/gix/tests/fixtures/make_shallow_repo.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -eu -o pipefail + +mkdir base +(cd base + git init -q + + git checkout -b main + touch a && git add a + git commit -q -m c1 + echo 1 >> a + git commit -q -am c2 + echo 1 >> a + git commit -q -am c3 +) + +mkdir empty +(cd empty + git init -q + + git checkout -b main + touch a && git add a + git commit -q -m c1 + touch .git/shallow +) + +git clone --depth 1 --bare file://$PWD/base shallow.git +git clone --depth 1 file://$PWD/base shallow diff --git a/gix/tests/repository/open.rs b/gix/tests/repository/open.rs index b0da3a2d022..f58ab5c9e5e 100644 --- a/gix/tests/repository/open.rs +++ b/gix/tests/repository/open.rs @@ -194,7 +194,8 @@ mod with_overrides { .set("GIT_SSL_VERSION", "tlsv1.3") .set("GIT_SSH_VARIANT", "ssh-variant-env") .set("GIT_SSH_COMMAND", "ssh-command-env") - .set("GIT_SSH", "ssh-command-fallback-env"); + .set("GIT_SSH", "ssh-command-fallback-env") + .set("GIT_SHALLOW_FILE", "shallow-file-env"); let mut opts = gix::open::Options::isolated() .cli_overrides([ "http.userAgent=agent-from-cli", @@ -206,6 +207,7 @@ mod with_overrides { "core.sshCommand=ssh-command-cli", "gitoxide.ssh.commandWithoutShellFallback=ssh-command-fallback-cli", "gitoxide.http.proxyAuthMethod=proxy-auth-method-cli", + "gitoxide.core.shallowFile=shallow-file-cli", ]) .config_overrides([ "http.userAgent=agent-from-api", @@ -217,6 +219,7 @@ mod with_overrides { "core.sshCommand=ssh-command-api", "gitoxide.ssh.commandWithoutShellFallback=ssh-command-fallback-api", "gitoxide.http.proxyAuthMethod=proxy-auth-method-api", + "gitoxide.core.shallowFile=shallow-file-api", ]); opts.permissions.env.git_prefix = Permission::Allow; opts.permissions.env.http_transport = Permission::Allow; @@ -229,6 +232,16 @@ mod with_overrides { "config always refers to the local one for safety" ); let config = repo.config_snapshot(); + assert_eq!( + config + .strings_by_key("gitoxide.core.shallowFile") + .expect("at least one value"), + [ + cow_bstr("shallow-file-cli"), + cow_bstr("shallow-file-api"), + cow_bstr("shallow-file-env") + ] + ); assert_eq!( config.strings_by_key("http.userAgent").expect("at least one value"), [ diff --git a/gix/tests/repository/state.rs b/gix/tests/repository/state.rs index 400a361d2ab..dcbeab12647 100644 --- a/gix/tests/repository/state.rs +++ b/gix/tests/repository/state.rs @@ -77,3 +77,25 @@ fn revert_sequence() -> Result { Ok(()) } + +mod shallow { + use crate::util::named_subrepo_opts; + + #[test] + fn without() -> crate::Result { + for name in ["base", "empty"] { + let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; + assert!(!repo.is_shallow()); + } + Ok(()) + } + + #[test] + fn with() -> crate::Result { + for name in ["shallow.git", "shallow"] { + let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; + assert!(repo.is_shallow()); + } + Ok(()) + } +} From 5bfbb9a32f8edb8bfb71ae00167277b9109de35a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 5 Mar 2023 16:51:11 +0100 Subject: [PATCH 3/4] feat: `Repository::shallow_commits()` returns an uptodate list of shallow boundary commits. --- gix/src/diff.rs | 17 +++++++ gix/src/lib.rs | 79 +++++---------------------------- gix/src/open/repository.rs | 1 + gix/src/permission.rs | 8 ++++ gix/src/permissions.rs | 1 + gix/src/prelude.rs | 4 ++ gix/src/progress.rs | 3 ++ gix/src/repository/impls.rs | 4 ++ gix/src/repository/init.rs | 3 ++ gix/src/repository/location.rs | 16 ------- gix/src/repository/mod.rs | 1 + gix/src/repository/shallow.rs | 58 ++++++++++++++++++++++++ gix/src/repository/state.rs | 7 --- gix/src/shallow.rs | 18 ++++++++ gix/src/state.rs | 25 +++++++++++ gix/src/types.rs | 2 + gix/tests/repository/mod.rs | 1 + gix/tests/repository/shallow.rs | 24 ++++++++++ gix/tests/repository/state.rs | 22 --------- 19 files changed, 180 insertions(+), 114 deletions(-) create mode 100644 gix/src/diff.rs create mode 100644 gix/src/permission.rs create mode 100644 gix/src/permissions.rs create mode 100644 gix/src/prelude.rs create mode 100644 gix/src/progress.rs create mode 100644 gix/src/repository/shallow.rs create mode 100644 gix/src/shallow.rs create mode 100644 gix/src/state.rs create mode 100644 gix/tests/repository/shallow.rs diff --git a/gix/src/diff.rs b/gix/src/diff.rs new file mode 100644 index 00000000000..b1081929394 --- /dev/null +++ b/gix/src/diff.rs @@ -0,0 +1,17 @@ +pub use gix_diff::*; + +/// +pub mod rename { + /// Determine how to do rename tracking. + #[derive(Debug, Copy, Clone, Eq, PartialEq)] + pub enum Tracking { + /// Do not track renames at all, the fastest option. + Disabled, + /// Track renames. + Renames, + /// Track renames and copies. + /// + /// This is the most expensive option. + RenamesAndCopies, + } +} diff --git a/gix/src/lib.rs b/gix/src/lib.rs index 257a613d73a..681c26bb078 100644 --- a/gix/src/lib.rs +++ b/gix/src/lib.rs @@ -98,12 +98,7 @@ pub mod interrupt; mod ext; /// -pub mod prelude { - pub use gix_features::parallel::reduce::Finalize; - pub use gix_odb::{Find, FindExt, Header, HeaderExt, Write}; - - pub use crate::ext::*; -} +pub mod prelude; /// pub mod path; @@ -133,31 +128,10 @@ mod repository; pub mod tag; /// -pub mod progress { - #[cfg(feature = "progress-tree")] - pub use gix_features::progress::prodash::tree; - pub use gix_features::progress::*; -} +pub mod progress; /// -pub mod diff { - pub use gix_diff::*; - /// - pub mod rename { - /// Determine how to do rename tracking. - #[derive(Debug, Copy, Clone, Eq, PartialEq)] - pub enum Tracking { - /// Do not track renames at all, the fastest option. - Disabled, - /// Track renames. - Renames, - /// Track renames and copies. - /// - /// This is the most expensive option. - RenamesAndCopies, - } - } -} +pub mod diff; /// See [ThreadSafeRepository::discover()], but returns a [`Repository`] instead. #[allow(clippy::result_large_err)] @@ -238,20 +212,10 @@ pub fn open_opts(directory: impl Into, options: open::Option } /// -pub mod permission { - /// - pub mod env_var { - /// - pub mod resource { - /// - pub type Error = gix_sec::permission::Error; - } - } -} +pub mod permission; + /// -pub mod permissions { - pub use crate::repository::permissions::{Config, Environment}; -} +pub mod permissions; pub use repository::permissions::Permissions; /// @@ -278,33 +242,10 @@ pub mod remote; pub mod init; /// Not to be confused with 'status'. -pub mod state { - /// Tell what operation is currently in progress. - #[derive(Debug, PartialEq, Eq)] - pub enum InProgress { - /// A mailbox is being applied. - ApplyMailbox, - /// A rebase is happening while a mailbox is being applied. - // TODO: test - ApplyMailboxRebase, - /// A git bisect operation has not yet been concluded. - Bisect, - /// A cherry pick operation. - CherryPick, - /// A cherry pick with multiple commits pending. - CherryPickSequence, - /// A merge operation. - Merge, - /// A rebase operation. - Rebase, - /// An interactive rebase operation. - RebaseInteractive, - /// A revert operation. - Revert, - /// A revert operation with multiple commits pending. - RevertSequence, - } -} +pub mod state; + +/// +pub mod shallow; /// pub mod discover; diff --git a/gix/src/open/repository.rs b/gix/src/open/repository.rs index 85dd91da711..6661d86f169 100644 --- a/gix/src/open/repository.rs +++ b/gix/src/open/repository.rs @@ -267,6 +267,7 @@ impl ThreadSafeRepository { // used when spawning new repositories off this one when following worktrees linked_worktree_options: options, index: gix_features::fs::MutableSnapshot::new().into(), + shallow_commits: gix_features::fs::MutableSnapshot::new().into(), }) } } diff --git a/gix/src/permission.rs b/gix/src/permission.rs new file mode 100644 index 00000000000..f74859def43 --- /dev/null +++ b/gix/src/permission.rs @@ -0,0 +1,8 @@ +/// +pub mod env_var { + /// + pub mod resource { + /// + pub type Error = gix_sec::permission::Error; + } +} diff --git a/gix/src/permissions.rs b/gix/src/permissions.rs new file mode 100644 index 00000000000..f64bb3bc2c6 --- /dev/null +++ b/gix/src/permissions.rs @@ -0,0 +1 @@ +pub use crate::repository::permissions::{Config, Environment}; diff --git a/gix/src/prelude.rs b/gix/src/prelude.rs new file mode 100644 index 00000000000..36fbfc7b186 --- /dev/null +++ b/gix/src/prelude.rs @@ -0,0 +1,4 @@ +pub use gix_features::parallel::reduce::Finalize; +pub use gix_odb::{Find, FindExt, Header, HeaderExt, Write}; + +pub use crate::ext::*; diff --git a/gix/src/progress.rs b/gix/src/progress.rs new file mode 100644 index 00000000000..0a88aa0441a --- /dev/null +++ b/gix/src/progress.rs @@ -0,0 +1,3 @@ +#[cfg(feature = "progress-tree")] +pub use gix_features::progress::prodash::tree; +pub use gix_features::progress::*; diff --git a/gix/src/repository/impls.rs b/gix/src/repository/impls.rs index 6cf2b2e9bc2..5da55290ca5 100644 --- a/gix/src/repository/impls.rs +++ b/gix/src/repository/impls.rs @@ -8,6 +8,7 @@ impl Clone for crate::Repository { self.config.clone(), self.options.clone(), self.index.clone(), + self.shallow_commits.clone(), ) } } @@ -40,6 +41,7 @@ impl From<&crate::ThreadSafeRepository> for crate::Repository { repo.config.clone(), repo.linked_worktree_options.clone(), repo.index.clone(), + repo.shallow_commits.clone(), ) } } @@ -54,6 +56,7 @@ impl From for crate::Repository { repo.config, repo.linked_worktree_options, repo.index, + repo.shallow_commits, ) } } @@ -68,6 +71,7 @@ impl From for crate::ThreadSafeRepository { config: r.config, linked_worktree_options: r.options, index: r.index, + shallow_commits: r.shallow_commits, } } } diff --git a/gix/src/repository/init.rs b/gix/src/repository/init.rs index ae6a42c3b0d..16659a01367 100644 --- a/gix/src/repository/init.rs +++ b/gix/src/repository/init.rs @@ -1,6 +1,7 @@ use std::cell::RefCell; impl crate::Repository { + #[allow(clippy::too_many_arguments)] pub(crate) fn from_refs_and_objects( refs: crate::RefStore, objects: crate::OdbHandle, @@ -9,6 +10,7 @@ impl crate::Repository { config: crate::config::Cache, linked_worktree_options: crate::open::Options, index: crate::worktree::IndexStorage, + shallow_commits: crate::shallow::CommitsStorage, ) -> Self { let objects = setup_objects(objects, &config); crate::Repository { @@ -20,6 +22,7 @@ impl crate::Repository { config, options: linked_worktree_options, index, + shallow_commits, } } diff --git a/gix/src/repository/location.rs b/gix/src/repository/location.rs index f64641f80d8..0bb8ea2535d 100644 --- a/gix/src/repository/location.rs +++ b/gix/src/repository/location.rs @@ -1,7 +1,5 @@ -use std::borrow::Cow; use std::path::PathBuf; -use crate::config::tree::{gitoxide, Key}; use gix_path::realpath::MAX_SYMLINKS; impl crate::Repository { @@ -43,20 +41,6 @@ impl crate::Repository { crate::path::install_dir() } - /// Return the path to the `shallow` file which contains hashes, one per line, that describe commits that don't have their - /// parents within this repository. - pub fn shallow_file(&self) -> PathBuf { - let shallow_name = self - .config - .resolved - .string_filter_by_key( - gitoxide::Core::SHALLOW_FILE.logical_name().as_str(), - &mut self.filter_config_section(), - ) - .unwrap_or(Cow::Borrowed("shallow".into())); - self.common_dir().join(gix_path::from_bstr(shallow_name)) - } - /// Returns the relative path which is the components between the working tree and the current working dir (CWD). /// Note that there may be `None` if there is no work tree, even though the `PathBuf` will be empty /// if the CWD is at the root of the work tree. diff --git a/gix/src/repository/mod.rs b/gix/src/repository/mod.rs index 31199e22d47..4fe3b27c622 100644 --- a/gix/src/repository/mod.rs +++ b/gix/src/repository/mod.rs @@ -30,6 +30,7 @@ pub(crate) mod permissions; mod reference; mod remote; mod revision; +mod shallow; mod snapshots; mod state; mod thread_safe; diff --git a/gix/src/repository/shallow.rs b/gix/src/repository/shallow.rs new file mode 100644 index 00000000000..408a3e4d6d0 --- /dev/null +++ b/gix/src/repository/shallow.rs @@ -0,0 +1,58 @@ +use crate::bstr::ByteSlice; +use crate::config::tree::{gitoxide, Key}; +use crate::Repository; +use std::borrow::Cow; +use std::path::PathBuf; + +impl Repository { + /// Return `true` if the repository is a shallow clone, i.e. contains history only up to a certain depth. + pub fn is_shallow(&self) -> bool { + self.shallow_file() + .metadata() + .map_or(false, |m| m.is_file() && m.len() > 0) + } + + /// Return a shared list of shallow commits which is updated automatically if the in-memory snapshot has become stale as the underlying file + /// on disk has changed. + /// + /// The shared list is shared across all clones of this repository. + pub fn shallow_commits(&self) -> Result, crate::shallow::open::Error> { + self.shallow_commits.recent_snapshot( + || self.shallow_file().metadata().ok().and_then(|m| m.modified().ok()), + || { + let buf = match std::fs::read(self.shallow_file()) { + Ok(buf) => buf, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err.into()), + }; + + let commits = buf + .lines() + .map(gix_hash::ObjectId::from_hex) + .collect::, _>>()?; + + if commits.is_empty() { + Ok(None) + } else { + Ok(Some(commits)) + } + }, + ) + } + + /// Return the path to the `shallow` file which contains hashes, one per line, that describe commits that don't have their + /// parents within this repository. + /// + /// Note that it may not exist if the repository isn't actually shallow. + pub fn shallow_file(&self) -> PathBuf { + let shallow_name = self + .config + .resolved + .string_filter_by_key( + gitoxide::Core::SHALLOW_FILE.logical_name().as_str(), + &mut self.filter_config_section(), + ) + .unwrap_or_else(|| Cow::Borrowed("shallow".into())); + self.common_dir().join(gix_path::from_bstr(shallow_name)) + } +} diff --git a/gix/src/repository/state.rs b/gix/src/repository/state.rs index c8619ca32c0..4034fe349db 100644 --- a/gix/src/repository/state.rs +++ b/gix/src/repository/state.rs @@ -41,11 +41,4 @@ impl crate::Repository { None } } - - /// Return `true` if the repository is a shallow clone, i.e. contains history only up to a certain depth. - pub fn is_shallow(&self) -> bool { - self.shallow_file() - .metadata() - .map_or(false, |m| m.is_file() && m.len() > 0) - } } diff --git a/gix/src/shallow.rs b/gix/src/shallow.rs new file mode 100644 index 00000000000..62e20e732ef --- /dev/null +++ b/gix/src/shallow.rs @@ -0,0 +1,18 @@ +pub(crate) type CommitsStorage = + gix_features::threading::OwnShared>>; +/// A lazily loaded and auto-updated list of commits which are at the shallow boundary (behind which there are no commits available), +/// sorted to allow bisecting. +pub type Commits = gix_features::fs::SharedSnapshot>; + +/// +pub mod open { + /// The error returned by [`Repository::shallow_commits()`][crate::Repository::shallow_commits()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("Could not open shallow file for reading")] + Io(#[from] std::io::Error), + #[error("Could not decode a line in shallow file as hex-encoded object hash")] + DecodeHash(#[from] gix_hash::decode::Error), + } +} diff --git a/gix/src/state.rs b/gix/src/state.rs new file mode 100644 index 00000000000..d8ee9983587 --- /dev/null +++ b/gix/src/state.rs @@ -0,0 +1,25 @@ +/// Tell what operation is currently in progress. +#[derive(Debug, PartialEq, Eq)] +pub enum InProgress { + /// A mailbox is being applied. + ApplyMailbox, + /// A rebase is happening while a mailbox is being applied. + // TODO: test + ApplyMailboxRebase, + /// A git bisect operation has not yet been concluded. + Bisect, + /// A cherry pick operation. + CherryPick, + /// A cherry pick with multiple commits pending. + CherryPickSequence, + /// A merge operation. + Merge, + /// A rebase operation. + Rebase, + /// An interactive rebase operation. + RebaseInteractive, + /// A revert operation. + Revert, + /// A revert operation with multiple commits pending. + RevertSequence, +} diff --git a/gix/src/types.rs b/gix/src/types.rs index 34ffdc8bf2e..eafa6a3f8a8 100644 --- a/gix/src/types.rs +++ b/gix/src/types.rs @@ -152,6 +152,7 @@ pub struct Repository { /// Particularly useful when following linked worktrees and instantiating new equally configured worktree repositories. pub(crate) options: crate::open::Options, pub(crate) index: crate::worktree::IndexStorage, + pub(crate) shallow_commits: crate::shallow::CommitsStorage, } /// An instance with access to everything a git repository entails, best imagined as container implementing `Sync + Send` for _most_ @@ -175,6 +176,7 @@ pub struct ThreadSafeRepository { pub(crate) linked_worktree_options: crate::open::Options, /// The index of this instances worktree. pub(crate) index: crate::worktree::IndexStorage, + pub(crate) shallow_commits: crate::shallow::CommitsStorage, } /// A remote which represents a way to interact with hosts for remote clones of the parent repository. diff --git a/gix/tests/repository/mod.rs b/gix/tests/repository/mod.rs index 079dbca69a1..3562662816c 100644 --- a/gix/tests/repository/mod.rs +++ b/gix/tests/repository/mod.rs @@ -5,6 +5,7 @@ mod object; mod open; mod reference; mod remote; +mod shallow; mod state; mod worktree; diff --git a/gix/tests/repository/shallow.rs b/gix/tests/repository/shallow.rs new file mode 100644 index 00000000000..41cc9f3256f --- /dev/null +++ b/gix/tests/repository/shallow.rs @@ -0,0 +1,24 @@ +use crate::util::{hex_to_id, named_subrepo_opts}; + +#[test] +fn no() -> crate::Result { + for name in ["base", "empty"] { + let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; + assert!(!repo.is_shallow()); + assert!(repo.shallow_commits()?.is_none()); + } + Ok(()) +} + +#[test] +fn yes() -> crate::Result { + for name in ["shallow.git", "shallow"] { + let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; + assert!(repo.is_shallow()); + assert_eq!( + repo.shallow_commits()?.expect("present").as_slice(), + [hex_to_id("30887839de28edf7ab66c860e5c58b4d445f6b12")] + ); + } + Ok(()) +} diff --git a/gix/tests/repository/state.rs b/gix/tests/repository/state.rs index dcbeab12647..400a361d2ab 100644 --- a/gix/tests/repository/state.rs +++ b/gix/tests/repository/state.rs @@ -77,25 +77,3 @@ fn revert_sequence() -> Result { Ok(()) } - -mod shallow { - use crate::util::named_subrepo_opts; - - #[test] - fn without() -> crate::Result { - for name in ["base", "empty"] { - let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; - assert!(!repo.is_shallow()); - } - Ok(()) - } - - #[test] - fn with() -> crate::Result { - for name in ["shallow.git", "shallow"] { - let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; - assert!(repo.is_shallow()); - } - Ok(()) - } -} From 4f8e952fc832c0240aabf2a8d7b1cfff1a5eeb62 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 5 Mar 2023 19:26:36 +0100 Subject: [PATCH 4/4] fix!: allow to traverse the entire commit graph of shallow repos Previously, when traversing commits, we would assume to be in a shallow repository if a commit's parent could not be found in the repository. Now we validate that assumption by reading the 'shallow' file to check if the last seen commit is on the commit boundary. This removes `is_shallow` and `error_on_missing_commit()` on the `revision::walk::Platform` as shallow commits are now known and handled without any guesswork. --- gix/src/repository/shallow.rs | 3 +- gix/src/revision/spec/parse/types.rs | 2 + gix/src/revision/walk.rs | 77 ++++++++---------- .../fixtures/generated-archives/.gitignore | 1 + .../fixtures/make_complex_shallow_repo.sh | 7 ++ gix/tests/repository/shallow.rs | 79 ++++++++++++++++++- 6 files changed, 124 insertions(+), 45 deletions(-) create mode 100644 gix/tests/fixtures/make_complex_shallow_repo.sh diff --git a/gix/src/repository/shallow.rs b/gix/src/repository/shallow.rs index 408a3e4d6d0..d6b8562ca5a 100644 --- a/gix/src/repository/shallow.rs +++ b/gix/src/repository/shallow.rs @@ -26,11 +26,12 @@ impl Repository { Err(err) => return Err(err.into()), }; - let commits = buf + let mut commits = buf .lines() .map(gix_hash::ObjectId::from_hex) .collect::, _>>()?; + commits.sort(); if commits.is_empty() { Ok(None) } else { diff --git a/gix/src/revision/spec/parse/types.rs b/gix/src/revision/spec/parse/types.rs index 4e523ab14fe..bd25d3b4e34 100644 --- a/gix/src/revision/spec/parse/types.rs +++ b/gix/src/revision/spec/parse/types.rs @@ -177,6 +177,8 @@ pub enum Error { }, #[error(transparent)] Traverse(#[from] gix_traverse::commit::ancestors::Error), + #[error(transparent)] + Walk(#[from] crate::revision::walk::Error), #[error("Spec does not contain a single object id")] SingleNotFound, } diff --git a/gix/src/revision/walk.rs b/gix/src/revision/walk.rs index 5b04b43a756..9a5ee7fc097 100644 --- a/gix/src/revision/walk.rs +++ b/gix/src/revision/walk.rs @@ -3,6 +3,16 @@ use gix_odb::FindExt; use crate::{revision, Repository}; +/// The error returned by [`Platform::all()`]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + AncestorIter(#[from] gix_traverse::commit::ancestors::Error), + #[error(transparent)] + ShallowCommits(#[from] crate::shallow::open::Error), +} + /// A platform to traverse the revision graph by adding starting points as well as points which shouldn't be crossed, /// returned by [`Repository::rev_walk()`]. pub struct Platform<'repo> { @@ -46,7 +56,7 @@ impl<'repo> Platform<'repo> { /// /// It's highly recommended to set an [`object cache`][Repository::object_cache_size()] on the parent repo /// to greatly speed up performance if the returned id is supposed to be looked up right after. - pub fn all(self) -> Result, gix_traverse::commit::ancestors::Error> { + pub fn all(self) -> Result, Error> { let Platform { repo, tips, @@ -56,16 +66,36 @@ impl<'repo> Platform<'repo> { Ok(revision::Walk { repo, inner: Box::new( - gix_traverse::commit::Ancestors::new( + gix_traverse::commit::Ancestors::filtered( tips, gix_traverse::commit::ancestors::State::default(), move |oid, buf| repo.objects.find_commit_iter(oid, buf), + { + let shallow_commits = repo.shallow_commits()?; + let mut grafted_parents_to_skip = Vec::new(); + let mut buf = Vec::new(); + move |id| match shallow_commits.as_ref() { + Some(commits) => { + let id = id.to_owned(); + if let Ok(idx) = grafted_parents_to_skip.binary_search(&id) { + grafted_parents_to_skip.remove(idx); + return false; + }; + if commits.binary_search(&id).is_ok() { + if let Ok(commit) = repo.objects.find_commit_iter(id, &mut buf) { + grafted_parents_to_skip.extend(commit.parent_ids()); + grafted_parents_to_skip.sort(); + } + }; + true + } + None => true, + } + }, ) .sorting(sorting)? .parents(parents), ), - is_shallow: None, - error_on_missing_commit: false, }) } } @@ -78,50 +108,13 @@ pub(crate) mod iter { pub(crate) repo: &'repo crate::Repository, pub(crate) inner: Box> + 'repo>, - pub(crate) error_on_missing_commit: bool, - // TODO: tests - /// After iteration this flag is true if the iteration was stopped prematurely due to missing parent commits. - /// Note that this flag won't be `Some` if any iteration error occurs, which is the case if - /// [`error_on_missing_commit()`][Walk::error_on_missing_commit()] was called. - /// - /// This happens if a repository is a shallow clone. - /// Note that this value is `None` as long as the iteration isn't complete. - pub is_shallow: Option, - } - - impl<'repo> Walk<'repo> { - // TODO: tests - /// Once invoked, the iteration will return an error if a commit cannot be found in the object database. This typically happens - /// when operating on a shallow clone and thus is non-critical by default. - /// - /// Check the [`is_shallow`][Walk::is_shallow] field once the iteration ended otherwise to learn if a shallow commit graph - /// was encountered. - pub fn error_on_missing_commit(mut self) -> Self { - self.error_on_missing_commit = true; - self - } } impl<'repo> Iterator for Walk<'repo> { type Item = Result, gix_traverse::commit::ancestors::Error>; fn next(&mut self) -> Option { - match self.inner.next() { - None => { - self.is_shallow = Some(false); - None - } - Some(Ok(oid)) => Some(Ok(oid.attach(self.repo))), - Some(Err(err @ gix_traverse::commit::ancestors::Error::FindExisting { .. })) => { - if self.error_on_missing_commit { - Some(Err(err)) - } else { - self.is_shallow = Some(true); - None - } - } - Some(Err(err)) => Some(Err(err)), - } + self.inner.next().map(|res| res.map(|id| id.attach(self.repo))) } } } diff --git a/gix/tests/fixtures/generated-archives/.gitignore b/gix/tests/fixtures/generated-archives/.gitignore index e046132fe5e..7cf4c0941d5 100644 --- a/gix/tests/fixtures/generated-archives/.gitignore +++ b/gix/tests/fixtures/generated-archives/.gitignore @@ -1,6 +1,7 @@ /make_worktree_repo.tar.xz /make_worktree_repo_with_configs.tar.xz /make_remote_repos.tar.xz +/make_complex_shallow_repo.tar.xz /make_fetch_repos.tar.xz /make_core_worktree_repo.tar.xz /make_signatures_repo.tar.xz diff --git a/gix/tests/fixtures/make_complex_shallow_repo.sh b/gix/tests/fixtures/make_complex_shallow_repo.sh new file mode 100644 index 00000000000..739b1bed75f --- /dev/null +++ b/gix/tests/fixtures/make_complex_shallow_repo.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eu -o pipefail + +remote="${1:?First argument is the complex repo to clone from}" + +git clone --depth 3 file://"$remote" shallow +git clone --depth 3 --bare file://"$remote" shallow.git diff --git a/gix/tests/repository/shallow.rs b/gix/tests/repository/shallow.rs index 41cc9f3256f..1243691b605 100644 --- a/gix/tests/repository/shallow.rs +++ b/gix/tests/repository/shallow.rs @@ -3,9 +3,20 @@ use crate::util::{hex_to_id, named_subrepo_opts}; #[test] fn no() -> crate::Result { for name in ["base", "empty"] { - let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; + let repo = named_subrepo_opts("make_shallow_repo.sh", name, crate::restricted())?; assert!(!repo.is_shallow()); assert!(repo.shallow_commits()?.is_none()); + let commits: Vec<_> = repo.head_id()?.ancestors().all()?.collect::>()?; + let expected = if name == "base" { + vec![ + hex_to_id("30887839de28edf7ab66c860e5c58b4d445f6b12"), + hex_to_id("d8523dfd5a7aa16562fa1c3e1d3b4a4494f97876"), + hex_to_id("05dc291f5376cde200316cb0b74b00cfebc79ea4"), + ] + } else { + vec![hex_to_id("05dc291f5376cde200316cb0b74b00cfebc79ea4")] + }; + assert_eq!(commits, expected); } Ok(()) } @@ -13,7 +24,7 @@ fn no() -> crate::Result { #[test] fn yes() -> crate::Result { for name in ["shallow.git", "shallow"] { - let repo = named_subrepo_opts("make_shallow_repo.sh", name, gix::open::Options::isolated())?; + let repo = named_subrepo_opts("make_shallow_repo.sh", name, crate::restricted())?; assert!(repo.is_shallow()); assert_eq!( repo.shallow_commits()?.expect("present").as_slice(), @@ -22,3 +33,67 @@ fn yes() -> crate::Result { } Ok(()) } + +mod traverse { + use crate::util::{hex_to_id, named_subrepo_opts}; + use gix_traverse::commit::Sorting; + + #[test] + fn boundary_is_detected_triggering_no_error() -> crate::Result { + for name in ["shallow.git", "shallow"] { + let repo = named_subrepo_opts("make_shallow_repo.sh", name, crate::restricted())?; + let commits: Vec<_> = repo.head_id()?.ancestors().all()?.collect::>()?; + assert_eq!(commits, [hex_to_id("30887839de28edf7ab66c860e5c58b4d445f6b12")]); + } + Ok(()) + } + + #[test] + fn complex_graphs_can_be_iterated_despite_multiple_shallow_boundaries() -> crate::Result { + let base = gix_path::realpath(gix_testtools::scripted_fixture_read_only("make_remote_repos.sh")?.join("base"))?; + let shallow_base = gix_testtools::scripted_fixture_read_only_with_args( + "make_complex_shallow_repo.sh", + Some(base.to_string_lossy()), + )?; + for name in ["shallow.git", "shallow"] { + let repo = gix::open_opts(shallow_base.join(name), crate::restricted())?; + assert_eq!( + repo.shallow_commits()?.expect("present").as_slice(), + [ + hex_to_id("27e71576a6335294aa6073ab767f8b36bdba81d0"), + hex_to_id("82024b2ef7858273337471cbd1ca1cedbdfd5616"), + hex_to_id("b5152869aedeb21e55696bb81de71ea1bb880c85"), + ] + ); + let commits: Vec<_> = repo + .head_id()? + .ancestors() + .sorting(Sorting::ByCommitTimeNewestFirst) + .all()? + .collect::>()?; + assert_eq!( + commits, + [ + "f99771fe6a1b535783af3163eba95a927aae21d5", + "2d9d136fb0765f2e24c44a0f91984318d580d03b", + "dfd0954dabef3b64f458321ef15571cc1a46d552", + "b5152869aedeb21e55696bb81de71ea1bb880c85", + "27e71576a6335294aa6073ab767f8b36bdba81d0", + "82024b2ef7858273337471cbd1ca1cedbdfd5616", + ] + .into_iter() + .map(hex_to_id) + .collect::>() + ); + + // should be + // * f99771f - (HEAD -> main, origin/main, origin/HEAD) A (18 years ago) + // | * 2d9d136 - C (18 years ago) + // *-. | dfd0954 - (tag: b-tag) B (18 years ago) + // | | * 27e7157 - (grafted) F (18 years ago) + // | * b515286 - (grafted) E (18 years ago) + // * 82024b2 - (grafted) D (18 years ago) + } + Ok(()) + } +}