diff --git a/Cargo.lock b/Cargo.lock index 954f648095b..6cb9c1c95da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1269,6 +1269,8 @@ dependencies = [ "gix-url", "itertools", "jwalk", + "layout-rs", + "open", "rusqlite", "serde", "serde_json", @@ -1874,7 +1876,7 @@ dependencies = [ "gix-object 0.30.0", "gix-odb", "gix-ref 0.30.0", - "gix-revision", + "gix-revwalk", "gix-testtools", "smallvec", "thiserror", @@ -2789,6 +2791,15 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + [[package]] name = "is-terminal" version = "0.4.7" @@ -2801,6 +2812,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + [[package]] name = "is_ci" version = "1.1.1" @@ -2869,6 +2890,12 @@ dependencies = [ "log", ] +[[package]] +name = "layout-rs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1164ef87cb9607c2d887216eca79f0fc92895affe1789bba805dd38d829584e0" + [[package]] name = "lazy_static" version = "1.4.0" @@ -3163,6 +3190,16 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "open" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16814a067484415fda653868c9be0ac5f2abd2ef5d951082a5f2fe1b3662944" +dependencies = [ + "is-wsl", + "pathdiff", +] + [[package]] name = "openssl" version = "0.10.51" @@ -3278,6 +3315,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "percent-encoding" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index 53cb0f39d4f..5930c96168e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ resolver = "2" [[bin]] name = "ein" +doc = false path = "src/ein.rs" test = false doctest = false @@ -19,6 +20,7 @@ doctest = false [[bin]] name = "gix" path = "src/gix.rs" +doc = false test = false doctest = false @@ -182,7 +184,7 @@ sha1_smol = { opt-level = 3 } [profile.release] overflow-checks = false -lto = "fat" +#lto = "fat" # this bloats files but assures destructors are called, important for tempfiles. One day I hope we # can wire up the 'abrt' signal handler so tempfiles will be removed in case of panics. panic = 'unwind' diff --git a/gitoxide-core/Cargo.toml b/gitoxide-core/Cargo.toml index 5f26cdc2a7a..3dc0dd5ed63 100644 --- a/gitoxide-core/Cargo.toml +++ b/gitoxide-core/Cargo.toml @@ -69,6 +69,10 @@ smallvec = { version = "1.10.0", optional = true } # for 'query' rusqlite = { version = "0.29.0", optional = true, features = ["bundled"] } +# for svg graph output +layout-rs = "0.1.1" +open = "4.1.0" + document-features = { version = "0.2.0", optional = true } [package.metadata.docs.rs] diff --git a/gitoxide-core/src/repository/clone.rs b/gitoxide-core/src/repository/clone.rs index 3a29fd7138f..7e0530ef899 100644 --- a/gitoxide-core/src/repository/clone.rs +++ b/gitoxide-core/src/repository/clone.rs @@ -88,14 +88,12 @@ pub(crate) mod function { } match fetch_outcome.status { - Status::NoPackReceived { .. } => { + Status::NoPackReceived { dry_run, .. } => { + assert!(!dry_run, "dry-run unsupported"); writeln!(err, "The cloned repository appears to be empty")?; } - Status::DryRun { .. } => unreachable!("dry-run unsupported"), Status::Change { - update_refs, - negotiation_rounds, - .. + update_refs, negotiate, .. } => { let remote = repo .find_default_remote(gix::remote::Direction::Fetch) @@ -103,7 +101,7 @@ pub(crate) mod function { let ref_specs = remote.refspecs(gix::remote::Direction::Fetch); print_updates( &repo, - negotiation_rounds, + &negotiate, update_refs, ref_specs, fetch_outcome.ref_map, diff --git a/gitoxide-core/src/repository/fetch.rs b/gitoxide-core/src/repository/fetch.rs index d2139b9415e..72c1f0c5a07 100644 --- a/gitoxide-core/src/repository/fetch.rs +++ b/gitoxide-core/src/repository/fetch.rs @@ -10,6 +10,8 @@ pub struct Options { pub ref_specs: Vec, pub shallow: gix::remote::fetch::Shallow, pub handshake_info: bool, + pub negotiation_info: bool, + pub open_negotiation_graph: Option, } pub const PROGRESS_RANGE: std::ops::RangeInclusive = 1..=3; @@ -17,6 +19,11 @@ pub const PROGRESS_RANGE: std::ops::RangeInclusive = 1..=3; pub(crate) mod function { use anyhow::bail; use gix::{prelude::ObjectIdExt, refspec::match_group::validate::Fix, remote::fetch::Status}; + use layout::backends::svg::SVGWriter; + use layout::core::base::Orientation; + use layout::core::geometry::Point; + use layout::core::style::StyleAttr; + use layout::std_shapes::shapes::{Arrow, Element, ShapeKind}; use super::Options; use crate::OutputFormat; @@ -31,6 +38,8 @@ pub(crate) mod function { dry_run, remote, handshake_info, + negotiation_info, + open_negotiation_graph, shallow, ref_specs, }: Options, @@ -62,41 +71,49 @@ pub(crate) mod function { let ref_specs = remote.refspecs(gix::remote::Direction::Fetch); match res.status { - Status::NoPackReceived { update_refs } => { - print_updates(&repo, 1, update_refs, ref_specs, res.ref_map, &mut out, err) - } - Status::DryRun { - update_refs, - negotiation_rounds, - } => print_updates( - &repo, - negotiation_rounds, - update_refs, - ref_specs, - res.ref_map, - &mut out, - err, - ), - Status::Change { + Status::NoPackReceived { update_refs, - write_pack_bundle, - negotiation_rounds, + negotiate, + dry_run: _, } => { + let negotiate_default = Default::default(); print_updates( &repo, - negotiation_rounds, + negotiate.as_ref().unwrap_or(&negotiate_default), update_refs, ref_specs, res.ref_map, &mut out, err, )?; + if negotiation_info { + print_negotiate_info(&mut out, negotiate.as_ref())?; + } + if let Some((negotiate, path)) = + open_negotiation_graph.and_then(|path| negotiate.as_ref().map(|n| (n, path))) + { + render_graph(&repo, &negotiate.graph, &path, progress)?; + } + Ok::<_, anyhow::Error>(()) + } + Status::Change { + update_refs, + write_pack_bundle, + negotiate, + } => { + print_updates(&repo, &negotiate, update_refs, ref_specs, res.ref_map, &mut out, err)?; if let Some(data_path) = write_pack_bundle.data_path { writeln!(out, "pack file: \"{}\"", data_path.display()).ok(); } if let Some(index_path) = write_pack_bundle.index_path { writeln!(out, "index file: \"{}\"", index_path.display()).ok(); } + if negotiation_info { + print_negotiate_info(&mut out, Some(&negotiate))?; + } + if let Some(path) = open_negotiation_graph { + render_graph(&repo, &negotiate.graph, &path, progress)?; + } Ok(()) } }?; @@ -106,9 +123,83 @@ pub(crate) mod function { Ok(()) } + fn render_graph( + repo: &gix::Repository, + graph: &gix::negotiate::IdMap, + path: &std::path::Path, + mut progress: impl gix::Progress, + ) -> anyhow::Result<()> { + progress.init(Some(graph.len()), gix::progress::count("commits")); + progress.set_name("building graph"); + + let mut map = gix::hashtable::HashMap::default(); + let mut vg = layout::topo::layout::VisualGraph::new(Orientation::TopToBottom); + + for (id, commit) in graph.iter().inspect(|_| progress.inc()) { + let source = match map.get(id) { + Some(handle) => *handle, + None => { + let handle = vg.add_node(new_node(id.attach(repo), commit.data.flags)); + map.insert(*id, handle); + handle + } + }; + + for parent_id in &commit.parents { + let dest = match map.get(parent_id) { + Some(handle) => *handle, + None => { + let flags = match graph.get(parent_id) { + Some(c) => c.data.flags, + None => continue, + }; + let dest = vg.add_node(new_node(parent_id.attach(repo), flags)); + map.insert(*parent_id, dest); + dest + } + }; + let arrow = Arrow::simple(""); + vg.add_edge(arrow, source, dest); + } + } + + let start = std::time::Instant::now(); + progress.set_name("layout graph"); + progress.info(format!("writing {path:?}…")); + let mut svg = SVGWriter::new(); + vg.do_it(false, false, false, &mut svg); + std::fs::write(path, svg.finalize().as_bytes())?; + open::that(path)?; + progress.show_throughput(start); + + return Ok(()); + + fn new_node(id: gix::Id<'_>, flags: gix::negotiate::Flags) -> Element { + let pt = Point::new(250., 50.); + let name = format!("{}\n\n{flags:?}", id.shorten_or_id()); + let shape = ShapeKind::new_box(name.as_str()); + let style = StyleAttr::simple(); + Element::create(shape, style, Orientation::LeftToRight, pt) + } + } + + fn print_negotiate_info( + mut out: impl std::io::Write, + negotiate: Option<&gix::remote::fetch::outcome::Negotiate>, + ) -> std::io::Result<()> { + writeln!(out, "Negotiation Phase Information")?; + match negotiate { + Some(negotiate) => { + writeln!(out, "\t{:?}", negotiate.rounds)?; + writeln!(out, "\tnum commits traversed in graph: {}", negotiate.graph.len()) + } + None => writeln!(out, "\tno negotiation performed"), + } + } + pub(crate) fn print_updates( repo: &gix::Repository, - negotiation_rounds: usize, + negotiate: &gix::remote::fetch::outcome::Negotiate, update_refs: gix::remote::fetch::refs::update::Outcome, refspecs: &[gix::refspec::RefSpec], mut map: gix::remote::fetch::RefMap, @@ -212,8 +303,10 @@ pub(crate) mod function { refspecs.len() )?; } - if negotiation_rounds != 1 { - writeln!(err, "needed {negotiation_rounds} rounds of pack-negotiation")?; + match negotiate.rounds.len() { + 0 => writeln!(err, "no negotiation was necessary")?, + 1 => {} + rounds => writeln!(err, "needed {rounds} rounds of pack-negotiation")?, } Ok(()) } diff --git a/gitoxide-core/src/repository/revision/list.rs b/gitoxide-core/src/repository/revision/list.rs index 91949503845..fc787a972c7 100644 --- a/gitoxide-core/src/repository/revision/list.rs +++ b/gitoxide-core/src/repository/revision/list.rs @@ -1,42 +1,141 @@ +use crate::OutputFormat; use std::ffi::OsString; +use std::path::PathBuf; -use anyhow::{bail, Context}; -use gix::traverse::commit::Sorting; +pub struct Context { + pub limit: Option, + pub spec: OsString, + pub format: OutputFormat, + pub text: Format, +} -use crate::OutputFormat; +pub enum Format { + Text, + Svg { path: PathBuf }, +} +pub const PROGRESS_RANGE: std::ops::RangeInclusive = 0..=2; -pub fn list( - mut repo: gix::Repository, - spec: OsString, - mut out: impl std::io::Write, - format: OutputFormat, -) -> anyhow::Result<()> { - if format != OutputFormat::Human { - bail!("Only human output is currently supported"); - } - repo.object_cache_size_if_unset(4 * 1024 * 1024); - - let spec = gix::path::os_str_into_bstr(&spec)?; - let id = repo - .rev_parse_single(spec) - .context("Only single revisions are currently supported")?; - let commits = id - .object()? - .peel_to_kind(gix::object::Kind::Commit) - .context("Need commitish as starting point")? - .id() - .ancestors() - .sorting(Sorting::ByCommitTimeNewestFirst) - .all()?; - for commit in commits { - let commit = commit?; - writeln!( - out, - "{} {} {}", - commit.id().shorten_or_id(), - commit.commit_time.expect("traversal with date"), - commit.parent_ids.len() - )?; +pub(crate) mod function { + use anyhow::{bail, Context}; + use gix::hashtable::HashMap; + use gix::traverse::commit::Sorting; + + use gix::Progress; + use layout::backends::svg::SVGWriter; + use layout::core::base::Orientation; + use layout::core::geometry::Point; + use layout::core::style::StyleAttr; + use layout::std_shapes::shapes::{Arrow, Element, ShapeKind}; + + use crate::repository::revision::list::Format; + use crate::OutputFormat; + + pub fn list( + mut repo: gix::Repository, + mut progress: impl Progress, + mut out: impl std::io::Write, + super::Context { + spec, + format, + text, + limit, + }: super::Context, + ) -> anyhow::Result<()> { + if format != OutputFormat::Human { + bail!("Only human output is currently supported"); + } + repo.object_cache_size_if_unset(4 * 1024 * 1024); + + let spec = gix::path::os_str_into_bstr(&spec)?; + let id = repo + .rev_parse_single(spec) + .context("Only single revisions are currently supported")?; + let commits = id + .object()? + .peel_to_kind(gix::object::Kind::Commit) + .context("Need commitish as starting point")? + .id() + .ancestors() + .sorting(Sorting::ByCommitTimeNewestFirst) + .all()?; + + let mut vg = match text { + Format::Svg { path } => ( + layout::topo::layout::VisualGraph::new(Orientation::TopToBottom), + path, + HashMap::default(), + ) + .into(), + Format::Text => None, + }; + progress.init(None, gix::progress::count("commits")); + progress.set_name("traverse"); + + let start = std::time::Instant::now(); + for commit in commits { + if gix::interrupt::is_triggered() { + bail!("interrupted by user"); + } + let commit = commit?; + match vg.as_mut() { + Some((vg, _path, map)) => { + let source = match map.get(&commit.id) { + Some(handle) => *handle, + None => { + let handle = vg.add_node(new_node(commit.id())); + map.insert(commit.id, handle); + handle + } + }; + + for parent_id in commit.parent_ids() { + let dest = match map.get(parent_id.as_ref()) { + Some(handle) => *handle, + None => { + let dest = vg.add_node(new_node(parent_id)); + map.insert(parent_id.detach(), dest); + dest + } + }; + let arrow = Arrow::simple(""); + vg.add_edge(arrow, source, dest); + } + } + None => { + writeln!( + out, + "{} {} {}", + commit.id().shorten_or_id(), + commit.commit_time.expect("traversal with date"), + commit.parent_ids.len() + )?; + } + } + progress.inc(); + if limit.map_or(false, |limit| limit == progress.step()) { + break; + } + } + + progress.show_throughput(start); + if let Some((mut vg, path, _)) = vg { + let start = std::time::Instant::now(); + progress.set_name("layout graph"); + progress.info(format!("writing {path:?}…")); + let mut svg = SVGWriter::new(); + vg.do_it(false, false, false, &mut svg); + std::fs::write(&path, svg.finalize().as_bytes())?; + open::that(path)?; + progress.show_throughput(start); + } + return Ok(()); + + fn new_node(id: gix::Id<'_>) -> Element { + let pt = Point::new(100., 30.); + let name = id.shorten_or_id().to_string(); + let shape = ShapeKind::new_box(name.as_str()); + let style = StyleAttr::simple(); + Element::create(shape, style, Orientation::LeftToRight, pt) + } } - Ok(()) } diff --git a/gitoxide-core/src/repository/revision/mod.rs b/gitoxide-core/src/repository/revision/mod.rs index 5e5dda98af5..e6366e88b59 100644 --- a/gitoxide-core/src/repository/revision/mod.rs +++ b/gitoxide-core/src/repository/revision/mod.rs @@ -1,5 +1,5 @@ -mod list; -pub use list::list; +pub mod list; +pub use list::function::list; mod explain; pub use explain::explain; diff --git a/gix-negotiate/Cargo.toml b/gix-negotiate/Cargo.toml index 928a976967f..23f5bf029c9 100644 --- a/gix-negotiate/Cargo.toml +++ b/gix-negotiate/Cargo.toml @@ -17,7 +17,7 @@ gix-hash = { version = "^0.11.2", path = "../gix-hash" } gix-object = { version = "^0.30.0", path = "../gix-object" } gix-date = { version = "^0.5.1", path = "../gix-date" } gix-commitgraph = { version = "^0.16.0", path = "../gix-commitgraph" } -gix-revision = { version = "^0.15.2", path = "../gix-revision" } +gix-revwalk = { version = "^0.1.0", path = "../gix-revwalk" } thiserror = "1.0.40" smallvec = "1.10.0" bitflags = "2" diff --git a/gix-negotiate/src/consecutive.rs b/gix-negotiate/src/consecutive.rs index 6a1504699e3..ae4b4e0306a 100644 --- a/gix-negotiate/src/consecutive.rs +++ b/gix-negotiate/src/consecutive.rs @@ -4,14 +4,14 @@ use gix_hash::ObjectId; use crate::{Error, Flags, Negotiator}; pub(crate) struct Algorithm { - revs: gix_revision::PriorityQueue, + revs: gix_revwalk::PriorityQueue, non_common_revs: usize, } impl Default for Algorithm { fn default() -> Self { Self { - revs: gix_revision::PriorityQueue::new(), + revs: gix_revwalk::PriorityQueue::new(), non_common_revs: 0, } } @@ -50,7 +50,7 @@ impl Algorithm { .try_lookup_or_insert_commit(id, |data| is_common = data.flags.contains(Flags::COMMON))? .filter(|_| !is_common) { - let mut queue = gix_revision::PriorityQueue::from_iter(Some((commit.commit_time, (id, 0_usize)))); + let mut queue = gix_revwalk::PriorityQueue::from_iter(Some((commit.commit_time, (id, 0_usize)))); if let Mark::ThisCommitAndAncestors = mode { commit.data.flags |= Flags::COMMON; if commit.data.flags.contains(Flags::SEEN) && !commit.data.flags.contains(Flags::POPPED) { diff --git a/gix-negotiate/src/lib.rs b/gix-negotiate/src/lib.rs index b05f944f888..571d2f1fe28 100644 --- a/gix-negotiate/src/lib.rs +++ b/gix-negotiate/src/lib.rs @@ -46,7 +46,7 @@ bitflags::bitflags! { /// Additional data to store with each commit when used by any of our algorithms. /// /// It's shared among those who use the [`Negotiator`] trait, and all implementations of it. -#[derive(Default, Copy, Clone)] +#[derive(Default, Debug, Copy, Clone)] pub struct Metadata { /// Used by `skipping`. /// Only used if commit is not COMMON @@ -58,7 +58,10 @@ pub struct Metadata { } /// The graph our callers use to store traversal information, for (re-)use in the negotiation implementation. -pub type Graph<'find> = gix_revision::Graph<'find, gix_revision::graph::Commit>; +pub type Graph<'find> = gix_revwalk::Graph<'find, gix_revwalk::graph::Commit>; + +/// A map associating an object id with its commit-metadata. +pub type IdMap = gix_revwalk::graph::IdMap>; /// The way the negotiation is performed. #[derive(Default, Debug, Copy, Clone, Eq, PartialEq)] @@ -141,4 +144,4 @@ pub trait Negotiator { } /// An error that happened during any of the methods on a [`Negotiator`]. -pub type Error = gix_revision::graph::lookup::commit::Error; +pub type Error = gix_revwalk::graph::lookup::commit::Error; diff --git a/gix-negotiate/src/skipping.rs b/gix-negotiate/src/skipping.rs index 28b8739198b..1969e05de4d 100644 --- a/gix-negotiate/src/skipping.rs +++ b/gix-negotiate/src/skipping.rs @@ -4,14 +4,14 @@ use gix_hash::ObjectId; use crate::{Error, Flags, Metadata, Negotiator}; pub(crate) struct Algorithm { - revs: gix_revision::PriorityQueue, + revs: gix_revwalk::PriorityQueue, non_common_revs: usize, } impl Default for Algorithm { fn default() -> Self { Self { - revs: gix_revision::PriorityQueue::new(), + revs: gix_revwalk::PriorityQueue::new(), non_common_revs: 0, } } @@ -41,7 +41,7 @@ impl Algorithm { })? .filter(|_| !is_common) { - let mut queue = gix_revision::PriorityQueue::from_iter(Some((commit.commit_time, id))); + let mut queue = gix_revwalk::PriorityQueue::from_iter(Some((commit.commit_time, id))); while let Some(id) = queue.pop_value() { if let Some(commit) = graph.try_lookup_or_insert_commit(id, |entry| { if !entry.flags.contains(Flags::POPPED) { diff --git a/gix-negotiate/tests/baseline/mod.rs b/gix-negotiate/tests/baseline/mod.rs index aec73f3c5a6..71d498c9a4e 100644 --- a/gix-negotiate/tests/baseline/mod.rs +++ b/gix-negotiate/tests/baseline/mod.rs @@ -59,7 +59,7 @@ fn run() -> crate::Result { let cache = use_cache .then(|| gix_commitgraph::at(store.store_ref().path().join("info")).ok()) .flatten(); - let mut graph = gix_revision::Graph::new( + let mut graph = gix_revwalk::Graph::new( |id, buf| { store .try_find(id, buf) diff --git a/gix-negotiate/tests/negotiate.rs b/gix-negotiate/tests/negotiate.rs index 357b54d615d..5abe81442be 100644 --- a/gix-negotiate/tests/negotiate.rs +++ b/gix-negotiate/tests/negotiate.rs @@ -39,7 +39,7 @@ mod baseline; #[test] fn size_of_entry() { assert_eq!( - std::mem::size_of::>(), + std::mem::size_of::>(), 56, "we may keep a lot of these, so let's not let them grow unnoticed" ); diff --git a/gix-revwalk/src/graph/mod.rs b/gix-revwalk/src/graph/mod.rs index d4290ec7e2a..2480469e148 100644 --- a/gix-revwalk/src/graph/mod.rs +++ b/gix-revwalk/src/graph/mod.rs @@ -5,6 +5,9 @@ use smallvec::SmallVec; use crate::Graph; +/// A mapping between an object id and arbitrary data, and produced when calling [`Graph::detach`]. +pub type IdMap = gix_hashtable::HashMap; + /// pub mod commit; @@ -20,7 +23,7 @@ pub type Generation = u32; impl<'find, T: std::fmt::Debug> std::fmt::Debug for Graph<'find, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Debug::fmt(&self.set, f) + std::fmt::Debug::fmt(&self.map, f) } } @@ -42,27 +45,27 @@ impl<'find, T: Default> Graph<'find, T> { impl<'find, T> Graph<'find, T> { /// Returns true if `id` has data associated with it, meaning that we processed it already. pub fn contains(&self, id: &gix_hash::oid) -> bool { - self.set.contains_key(id.as_ref()) + self.map.contains_key(id.as_ref()) } /// Returns the data associated with `id` if available. pub fn get(&self, id: &gix_hash::oid) -> Option<&T> { - self.set.get(id) + self.map.get(id) } /// Returns the data associated with `id` if available as mutable reference. pub fn get_mut(&mut self, id: &gix_hash::oid) -> Option<&mut T> { - self.set.get_mut(id) + self.map.get_mut(id) } /// Insert `id` into the graph and associate it with `value`, returning the previous value associated with it if it existed. pub fn insert(&mut self, id: gix_hash::ObjectId, value: T) -> Option { - self.set.insert(id, value) + self.map.insert(id, value) } /// Remove all data from the graph to start over. pub fn clear(&mut self) { - self.set.clear(); + self.map.clear(); } /// Insert the parents of commit named `id` to the graph and associate new parents with data @@ -80,7 +83,7 @@ impl<'find, T> Graph<'find, T> { let parents: SmallVec<[_; 2]> = commit.iter_parents().collect(); for parent_id in parents { let parent_id = parent_id?; - match self.set.entry(parent_id) { + match self.map.entry(parent_id) { gix_hashtable::hash_map::Entry::Vacant(entry) => { let parent = match try_lookup(&parent_id, &mut self.find, self.cache.as_ref(), &mut self.parent_buf) .map_err(|err| insert_parents::Error::Lookup(lookup::existing::Error::Find(err)))? @@ -102,6 +105,11 @@ impl<'find, T> Graph<'find, T> { } Ok(()) } + + /// Turn ourselves into the underlying graph structure, which is a mere mapping between object ids and their data. + pub fn detach(self) -> IdMap { + self.map + } } /// Initialization @@ -125,7 +133,7 @@ impl<'find, T> Graph<'find, T> { find(id, buf).map_err(|err| Box::new(err) as Box) }), cache: cache.into(), - set: gix_hashtable::HashMap::default(), + map: gix_hashtable::HashMap::default(), buf: Vec::new(), parent_buf: Vec::new(), } @@ -145,7 +153,7 @@ impl<'find, T> Graph<'find, Commit> { new_data: impl FnOnce() -> T, update_data: impl FnOnce(&mut T), ) -> Result>, lookup::commit::Error> { - match self.set.entry(id) { + match self.map.entry(id) { gix_hashtable::hash_map::Entry::Vacant(entry) => { let res = try_lookup(&id, &mut self.find, self.cache.as_ref(), &mut self.buf)?; let commit = match res { @@ -160,7 +168,7 @@ impl<'find, T> Graph<'find, Commit> { update_data(&mut entry.get_mut().data); } }; - Ok(self.set.get_mut(&id)) + Ok(self.map.get_mut(&id)) } } @@ -202,7 +210,7 @@ impl<'find, T> Graph<'find, T> { ) -> Result>, lookup::Error> { let res = try_lookup(&id, &mut self.find, self.cache.as_ref(), &mut self.buf)?; Ok(res.map(|commit| { - match self.set.entry(id) { + match self.map.entry(id) { gix_hashtable::hash_map::Entry::Vacant(entry) => { let mut data = default(); update_data(&mut data); @@ -255,7 +263,7 @@ impl<'a, 'find, T> Index<&'a gix_hash::oid> for Graph<'find, T> { type Output = T; fn index(&self, index: &'a oid) -> &Self::Output { - &self.set[index] + &self.map[index] } } diff --git a/gix-revwalk/src/lib.rs b/gix-revwalk/src/lib.rs index 44c5ca3e704..9348d9e162a 100644 --- a/gix-revwalk/src/lib.rs +++ b/gix-revwalk/src/lib.rs @@ -31,12 +31,13 @@ pub struct Graph<'find, T> { /// A way to speedup commit access, essentially a multi-file commit database. cache: Option, /// The set of cached commits that we have seen once, along with data associated with them. - set: gix_hashtable::HashMap, + map: graph::IdMap, /// A buffer for writing commit data into. buf: Vec, /// Another buffer we typically use to store parents. parent_buf: Vec, } + /// pub mod graph; diff --git a/gix/src/lib.rs b/gix/src/lib.rs index 5de702dbfa5..5808fa4eb61 100644 --- a/gix/src/lib.rs +++ b/gix/src/lib.rs @@ -77,6 +77,7 @@ pub use gix_features::{parallel, progress::Progress, threading}; pub use gix_fs as fs; pub use gix_glob as glob; pub use gix_hash as hash; +pub use gix_hashtable as hashtable; pub use gix_ignore as ignore; #[doc(inline)] pub use gix_index as index; diff --git a/gix/src/remote/connection/fetch/mod.rs b/gix/src/remote/connection/fetch/mod.rs index b4fe0093544..0ba73c3530b 100644 --- a/gix/src/remote/connection/fetch/mod.rs +++ b/gix/src/remote/connection/fetch/mod.rs @@ -46,27 +46,25 @@ pub enum Status { /// /// As we could determine that nothing changed without remote interaction, there was no negotiation at all. NoPackReceived { + /// If `true`, we didn't receive a pack due to dry-run mode being enabled. + dry_run: bool, + /// Information about the pack negotiation phase if negotiation happened at all. + /// + /// It's possible that negotiation didn't have to happen as no reference of interest changed on the server. + negotiate: Option, /// However, depending on the refspecs, references might have been updated nonetheless to point to objects as /// reported by the remote. update_refs: refs::update::Outcome, }, /// There was at least one tip with a new object which we received. Change { - /// The number of rounds it took to minimize the pack to contain only the objects we don't have. - negotiation_rounds: usize, + /// Information about the pack negotiation phase. + negotiate: outcome::Negotiate, /// Information collected while writing the pack and its index. write_pack_bundle: gix_pack::bundle::write::Outcome, /// Information collected while updating references. update_refs: refs::update::Outcome, }, - /// A dry run was performed which leaves the local repository without any change - /// nor will a pack have been received. - DryRun { - /// The number of rounds it took to minimize the *would-be-sent*-pack to contain only the objects we don't have. - negotiation_rounds: usize, - /// Information about what updates to refs would have been done. - update_refs: refs::update::Outcome, - }, } /// The outcome of receiving a pack via [`Prepare::receive()`]. @@ -78,6 +76,46 @@ pub struct Outcome { pub status: Status, } +/// Additional types related to the outcome of a fetch operation. +pub mod outcome { + /// Information about the negotiation phase of a fetch. + /// + /// Note that negotiation can happen even if no pack is ultimately produced. + #[derive(Default, Debug, Clone)] + pub struct Negotiate { + /// The negotiation graph indicating what kind of information 'the algorithm' collected in the end. + pub graph: gix_negotiate::IdMap, + /// Additional information for each round of negotiation. + pub rounds: Vec, + } + + /// + pub mod negotiate { + /// Key information about each round in the pack-negotiation. + #[derive(Debug, Clone)] + pub struct Round { + /// The amount of `HAVE` lines sent this round. + /// + /// Each `HAVE` is an object that we tell the server about which would acknowledge each one it has as well. + pub haves_sent: usize, + /// A total counter, over all previous rounds, indicating how many `HAVE`s we sent without seeing a single acknowledgement, + /// i.e. the indication of a common object. + /// + /// This number maybe zero or be lower compared to the previous round if we have received at least one acknowledgement. + pub in_vain: usize, + /// The amount of haves we should send in this round. + /// + /// If the value is lower than `haves_sent` (the `HAVE` lines actually sent), the negotiation algorithm has run out of options + /// which typically indicates the end of the negotiation phase. + pub haves_to_send: usize, + /// If `true`, the server reported, as response to our previous `HAVE`s, that at least one of them is in common by acknowledging it. + /// + /// This may also lead to the server responding with a pack. + pub previous_response_had_at_least_one_in_common: bool, + } + } +} + /// The progress ids used in during various steps of the fetch operation. /// /// Note that tagged progress isn't very widely available yet, but support can be improved as needed. diff --git a/gix/src/remote/connection/fetch/receive_pack.rs b/gix/src/remote/connection/fetch/receive_pack.rs index 7a938c55c40..ddb28203717 100644 --- a/gix/src/remote/connection/fetch/receive_pack.rs +++ b/gix/src/remote/connection/fetch/receive_pack.rs @@ -19,7 +19,8 @@ use crate::{ connection::fetch::config, fetch, fetch::{ - negotiate, negotiate::Algorithm, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Shallow, Status, + negotiate, negotiate::Algorithm, outcome, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, + Shallow, Status, }, }, Progress, Repository, @@ -141,11 +142,10 @@ where negotiate::make_refmapping_ignore_predicate(con.remote.fetch_tags, &self.ref_map), )?; let mut previous_response = None::; - let mut round = 1; - let mut write_pack_bundle = match &action { + let (mut write_pack_bundle, negotiate) = match &action { negotiate::Action::NoChange | negotiate::Action::SkipToRefUpdate => { gix_protocol::indicate_end_of_interaction(&mut con.transport).await.ok(); - None + (None, None) } negotiate::Action::MustNegotiate { remote_ref_target_known, @@ -158,6 +158,7 @@ where &self.shallow, negotiate::make_refmapping_ignore_predicate(con.remote.fetch_tags, &self.ref_map), ); + let mut rounds = Vec::new(); let is_stateless = arguments.is_stateless(!con.transport.connection_persists_across_multiple_requests()); let mut haves_to_send = gix_negotiate::window_size(is_stateless, None); @@ -166,7 +167,7 @@ where let mut common = is_stateless.then(Vec::new); let reader = 'negotiation: loop { progress.step(); - progress.set_name(format!("negotiate (round {round})")); + progress.set_name(format!("negotiate (round {})", rounds.len() + 1)); let is_done = match negotiate::one_round( negotiator.deref_mut(), @@ -182,6 +183,12 @@ where } seen_ack |= ack_seen; in_vain += haves_sent; + rounds.push(outcome::negotiate::Round { + haves_sent, + in_vain, + haves_to_send, + previous_response_had_at_least_one_in_common: ack_seen, + }); let is_done = haves_sent != haves_to_send || (seen_ack && in_vain >= 256); haves_to_send = gix_negotiate::window_size(is_stateless, haves_to_send); is_done @@ -206,11 +213,9 @@ where setup_remote_progress(progress, &mut reader, should_interrupt); } break 'negotiation reader; - } else { - round += 1; } }; - drop(graph); + let graph = graph.detach(); drop(graph_repo); let previous_response = previous_response.expect("knowledge of a pack means a response was received"); if !previous_response.shallow_updates().is_empty() && shallow_lock.is_none() { @@ -267,7 +272,7 @@ where crate::shallow::write(shallow_lock, shallow_commits, previous_response.shallow_updates())?; } } - write_pack_bundle + (write_pack_bundle, Some(outcome::Negotiate { graph, rounds })) } }; @@ -294,21 +299,17 @@ where let out = Outcome { ref_map: std::mem::take(&mut self.ref_map), - status: if matches!(self.dry_run, fetch::DryRun::Yes) { - assert!(write_pack_bundle.is_none(), "in dry run we never read a bundle"); - Status::DryRun { + status: match write_pack_bundle { + Some(write_pack_bundle) => Status::Change { + write_pack_bundle, update_refs, - negotiation_rounds: round, - } - } else { - match write_pack_bundle { - Some(write_pack_bundle) => Status::Change { - write_pack_bundle, - update_refs, - negotiation_rounds: round, - }, - None => Status::NoPackReceived { update_refs }, - } + negotiate: negotiate.expect("if we have a pack, we always negotiated it"), + }, + None => Status::NoPackReceived { + dry_run: matches!(self.dry_run, fetch::DryRun::Yes), + negotiate, + update_refs, + }, }, }; Ok(out) diff --git a/gix/src/remote/fetch.rs b/gix/src/remote/fetch.rs index f067b5d6c40..eb40403e5c3 100644 --- a/gix/src/remote/fetch.rs +++ b/gix/src/remote/fetch.rs @@ -11,7 +11,9 @@ pub mod negotiate { } #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -pub use super::connection::fetch::{prepare, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Status}; +pub use super::connection::fetch::{ + outcome, prepare, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, Status, +}; /// If `Yes`, don't really make changes but do as much as possible to get an idea of what would be done. #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/gix/tests/remote/fetch.rs b/gix/tests/remote/fetch.rs index 8f92fe8304a..9c8d50719be 100644 --- a/gix/tests/remote/fetch.rs +++ b/gix/tests/remote/fetch.rs @@ -137,12 +137,13 @@ mod blocking_and_async_io { match out.status { Status::Change { - negotiation_rounds, + negotiate, write_pack_bundle, .. } => { assert_eq!( - negotiation_rounds, 1, + negotiate.rounds.len(), + 1, "we don't really have a way to see that tips from alternates were added, I think" ); assert_eq!( @@ -215,11 +216,12 @@ mod blocking_and_async_io { match changes.status { Status::Change { write_pack_bundle, - negotiation_rounds, + negotiate, .. } => { assert_eq!( - negotiation_rounds, expected_negotiation_rounds, + negotiate.rounds.len(), + expected_negotiation_rounds, "we need multiple rounds" ); // the server only has our `b1` and an extra commit or two. @@ -389,8 +391,9 @@ mod blocking_and_async_io { .await?; match res.status { - fetch::Status::NoPackReceived { update_refs } => { + fetch::Status::NoPackReceived { update_refs, negotiate: _, dry_run } => { assert_eq!(update_refs.edits.len(), expected_ref_count, "{shallow_args:?}|{fetch_tags:?}"); + assert!(!dry_run, "we actually perform the operation"); }, _ => unreachable!( "{shallow_args:?}|{fetch_tags:?}: default negotiation is able to realize nothing is required and doesn't get to receiving a pack" @@ -446,8 +449,8 @@ mod blocking_and_async_io { .await?; match res.status { - gix::remote::fetch::Status::Change { write_pack_bundle, update_refs, negotiation_rounds } => { - assert_eq!(negotiation_rounds, 1); + gix::remote::fetch::Status::Change { write_pack_bundle, update_refs, negotiate } => { + assert_eq!(negotiate.rounds.len(), 1); assert_eq!(write_pack_bundle.index.data_hash, hex_to_id(expected_data_hash), ); assert_eq!(write_pack_bundle.index.num_objects, 3 + num_objects_offset, "{fetch_tags:?}"); assert!(write_pack_bundle.data_path.as_deref().map_or(false, std::path::Path::is_file)); @@ -530,9 +533,9 @@ mod blocking_and_async_io { fetch::Status::Change { write_pack_bundle, update_refs, - negotiation_rounds, + negotiate, } => { - assert_eq!(negotiation_rounds, 1); + assert_eq!(negotiate.rounds.len(), 1); assert_eq!(write_pack_bundle.pack_version, gix::odb::pack::data::Version::V2); assert_eq!(write_pack_bundle.object_hash, repo.object_hash()); assert_eq!(write_pack_bundle.index.num_objects, 4, "{dry_run}: this value is 4 when git does it with 'consecutive' negotiation style, but could be 33 if completely naive."); @@ -568,16 +571,17 @@ mod blocking_and_async_io { update_refs } - fetch::Status::DryRun { + fetch::Status::NoPackReceived { + dry_run, update_refs, - negotiation_rounds, + negotiate: _, } => { - assert_eq!(negotiation_rounds, 1); + assert!( + dry_run, + "the only reason we receive no pack is if we are in dry-run mode" + ); update_refs } - fetch::Status::NoPackReceived { .. } => { - unreachable!("we firmly expect changes here, as the other origin has changes") - } }; assert_eq!( diff --git a/src/ein.rs b/src/ein.rs index c056f025a25..bfb4ba4716d 100644 --- a/src/ein.rs +++ b/src/ein.rs @@ -1,9 +1,3 @@ -//! ## Feature Flags -#![cfg_attr( - feature = "document-features", - cfg_attr(doc, doc = ::document_features::document_features!()) -)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![deny(rust_2018_idioms, unsafe_code)] mod porcelain; diff --git a/src/gix.rs b/src/gix.rs index 7447a51a923..7e78175060b 100644 --- a/src/gix.rs +++ b/src/gix.rs @@ -1,10 +1,3 @@ -//! The `gitoxide` plumbing. -//! ## Feature Flags -#![cfg_attr( - feature = "document-features", - cfg_attr(doc, doc = ::document_features::document_features!()) -)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![deny(unsafe_code, rust_2018_idioms)] mod plumbing; diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs index 7896e6e66cd..115253dd178 100644 --- a/src/plumbing/main.rs +++ b/src/plumbing/main.rs @@ -189,6 +189,8 @@ pub fn main() -> Result<()> { Subcommands::Fetch(crate::plumbing::options::fetch::Platform { dry_run, handshake_info, + negotiation_info, + open_negotiation_graph, remote, shallow, ref_spec, @@ -198,6 +200,8 @@ pub fn main() -> Result<()> { dry_run, remote, handshake_info, + negotiation_info, + open_negotiation_graph, shallow: shallow.into(), ref_specs: ref_spec, }; @@ -693,14 +697,26 @@ pub fn main() -> Result<()> { }, ), Subcommands::Revision(cmd) => match cmd { - revision::Subcommands::List { spec } => prepare_and_run( + revision::Subcommands::List { spec, svg, limit } => prepare_and_run( "revision-list", - verbose, + auto_verbose, progress, progress_keep_open, - None, - move |_progress, out, _err| { - core::repository::revision::list(repository(Mode::Lenient)?, spec, out, format) + core::repository::revision::list::PROGRESS_RANGE, + move |progress, out, _err| { + core::repository::revision::list( + repository(Mode::Lenient)?, + progress, + out, + core::repository::revision::list::Context { + limit, + spec, + format, + text: svg.map_or(core::repository::revision::list::Format::Text, |path| { + core::repository::revision::list::Format::Svg { path } + }), + }, + ) }, ), revision::Subcommands::PreviousBranches => prepare_and_run( diff --git a/src/plumbing/options/mod.rs b/src/plumbing/options/mod.rs index 3f0a9a27053..4a9ce607ae2 100644 --- a/src/plumbing/options/mod.rs +++ b/src/plumbing/options/mod.rs @@ -155,6 +155,14 @@ pub mod fetch { #[clap(long, short = 'H')] pub handshake_info: bool, + /// Print statistics about negotiation phase. + #[clap(long, short = 's')] + pub negotiation_info: bool, + + /// Open the commit graph used for negotiation and write an SVG file to PATH. + #[clap(long, value_name = "PATH", short = 'g')] + pub open_negotiation_graph: Option, + #[clap(flatten)] pub shallow: ShallowOptions, @@ -441,6 +449,12 @@ pub mod revision { /// List all commits reachable from the given rev-spec. #[clap(visible_alias = "l")] List { + /// How many commits to list at most. + #[clap(long, short = 'l')] + limit: Option, + /// Write the graph as SVG file to the given path. + #[clap(long, short = 's')] + svg: Option, /// The rev-spec to list reachable commits from. #[clap(default_value = "@")] spec: std::ffi::OsString,