From 2b4b7c7ce09a609c1295f83354fc9792913c31a3 Mon Sep 17 00:00:00 2001 From: Mshehu5 Date: Tue, 17 Nov 2026 20:23:41 +0100 Subject: [PATCH 1/3] Simplify repository setup Replace the DiscoverRepository struct, which only had a run method, with a plain discover_repository function. Drop the MissingRevisions check: clap already requires a revision or --all. --- src/lib.rs | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 786adf4..0d48bba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,10 +7,7 @@ use std::path::PathBuf; use std::process::ExitStatus; pub fn run_revisions(revisions: &[&str], all: bool, output: &mut impl Write) -> Result<(), Error> { - if revisions.is_empty() && !all { - return Err(Error::MissingRevisions); - } - let repository = DiscoverRepository.run()?; + let repository = discover_repository()?; let commits = history::resolve_commits(&repository.git_repo, revisions, all)?; if commits.is_empty() { @@ -32,18 +29,14 @@ struct Repository { root: PathBuf, } -struct DiscoverRepository; - -impl DiscoverRepository { - fn run(self) -> Result { - let git_repo = git2::Repository::discover(".").map_err(Error::DiscoverRepository)?; - let root = git_repo - .workdir() - .ok_or(Error::BareRepository)? - .canonicalize() - .map_err(Error::CanonicalizeRepository)?; - Ok(Repository { git_repo, root }) - } +fn discover_repository() -> Result { + let git_repo = git2::Repository::discover(".").map_err(Error::DiscoverRepository)?; + let root = git_repo + .workdir() + .ok_or(Error::BareRepository)? + .canonicalize() + .map_err(Error::CanonicalizeRepository)?; + Ok(Repository { git_repo, root }) } #[derive(Clone, Debug, Eq, PartialEq)] @@ -57,8 +50,6 @@ impl fmt::Display for CommitId { #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("provide at least one revision or select all refs")] - MissingRevisions, #[error("could not discover the Git repository: {0}")] DiscoverRepository(#[source] git2::Error), #[error("cannot validate a bare Git repository")] From 2f9112a4531354bd2efcc11e4523dd5a389b9d62 Mon Sep 17 00:00:00 2001 From: Mshehu5 Date: Tue, 17 Nov 2026 20:24:39 +0100 Subject: [PATCH 2/3] Limit revisions to X and A..B A shallow clone may be missing the parents or common ancestors that other revision forms need. To make a command behave the same in CI and locally, accept only X and A..B in every clone, and say so when another form is given. This removes --all, A...B, ^X, X^! and X^@, which came in with #4, and their tests. The CLI tests use A..B or a plain X instead. --- src/history.rs | 253 ++++++++++++++----------------------------------- src/lib.rs | 10 +- src/main.rs | 10 +- tests/cli.rs | 4 +- 4 files changed, 83 insertions(+), 194 deletions(-) diff --git a/src/history.rs b/src/history.rs index e01229d..30cb57a 100644 --- a/src/history.rs +++ b/src/history.rs @@ -1,110 +1,70 @@ -use git2::{ErrorCode, Object, Oid, Repository, RevparseMode, Revwalk, Sort}; +use git2::{Oid, Repository, RevparseMode, Sort}; use crate::{CommitId, Error}; +// Find the commits to validate: reachable from an included revision but +// not from an excluded one, with parents before their children. pub(crate) fn resolve_commits( repo: &Repository, revisions: &[&str], - all: bool, ) -> Result, Error> { - if !all - && let [input] = revisions - && let Some(revision) = input.strip_suffix("^!") - { - // A single-commit selection does not need any parent history. - return Ok(vec![CommitId(resolve(repo, revision)?.id())]); - } - - let mut walk = repo.revwalk().map_err(Error::WalkHistory)?; - walk.set_sorting(Sort::TOPOLOGICAL | Sort::REVERSE) - .map_err(Error::WalkHistory)?; - - if all { - walk.push_glob("refs/*").map_err(Error::WalkHistory)?; - match repo.head() { - Ok(head) => { - let commit = head.peel_to_commit().map_err(Error::PeelToCommit)?; - walk.push(commit.id()).map_err(Error::WalkHistory)?; - } - // An empty repository has no HEAD commit to include. - Err(error) if error.code() == ErrorCode::UnbornBranch => {} - Err(error) => return Err(Error::ParseRevision(error)), - } - } - for input in revisions { - add_revision(repo, &mut walk, input)?; - } + let parsed = parse_revisions(repo, revisions)?; + let selected = walk(repo, &parsed.include, &parsed.exclude)?; + Ok(selected.into_iter().map(CommitId).collect()) +} - walk.map(|id| id.map(CommitId).map_err(Error::WalkHistory)) - .collect() +// The commits named by the arguments, split into included and excluded. +struct Revisions { + include: Vec, + exclude: Vec, } -fn add_revision(repo: &Repository, walk: &mut Revwalk<'_>, input: &str) -> Result<(), Error> { - // libgit2 parses single revisions and ranges. Expand the remaining - // revision-set shorthands into its native push/hide operations. - if let Some(revision) = input.strip_prefix('^') { - walk.hide(resolve(repo, revision)?.id()) - .map_err(Error::WalkHistory)?; - } else if let Some(revision) = input.strip_suffix("^!") { - let commit = resolve(repo, revision)?; - walk.push(commit.id()).map_err(Error::WalkHistory)?; - for parent in commit.parent_ids() { - walk.hide(parent).map_err(Error::WalkHistory)?; - } - } else if let Some(revision) = input.strip_suffix("^@") { - for parent in resolve(repo, revision)?.parent_ids() { - walk.push(parent).map_err(Error::WalkHistory)?; - } - } else { +// Accept the two forms libgit2's revision parser returns that only include +// or exclude history: X and A..B. A...B needs common ancestors, which a +// shallow clone may not have downloaded, so it is rejected in every clone. +fn parse_revisions(repo: &Repository, revisions: &[&str]) -> Result { + let mut include = Vec::new(); + let mut exclude = Vec::new(); + for input in revisions { let spec = repo.revparse(input).map_err(Error::ParseRevision)?; - let from = peel(spec.from().expect("libgit2 supplies the range start"))?; - match spec.mode() { - mode if mode.contains(RevparseMode::SINGLE) => { - walk.push(from).map_err(Error::WalkHistory)?; - } - mode if mode.contains(RevparseMode::MERGE_BASE) => { - let to = peel(spec.to().expect("libgit2 supplies the range end"))?; - walk.push(from).map_err(Error::WalkHistory)?; - walk.push(to).map_err(Error::WalkHistory)?; - match repo.merge_bases(from, to) { - Ok(bases) => { - for base in bases.iter() { - walk.hide(*base).map_err(Error::WalkHistory)?; - } - } - // Unrelated histories have no common ancestors to hide. - Err(error) if error.code() == ErrorCode::NotFound => {} - Err(error) => return Err(Error::MergeBases(error)), - } - } - mode if mode.contains(RevparseMode::RANGE) => { - let to = peel(spec.to().expect("libgit2 supplies the range end"))?; - walk.hide(from).map_err(Error::WalkHistory)?; - walk.push(to).map_err(Error::WalkHistory)?; - } - mode => return Err(Error::UnsupportedRevisionMode(mode)), + let commit = |object: Option<&git2::Object<'_>>| { + object + .expect("libgit2 sets both ends of a range") + .peel_to_commit() + .map(|commit| commit.id()) + .map_err(Error::PeelToCommit) + }; + let mode = spec.mode(); + if mode.contains(RevparseMode::MERGE_BASE) { + return Err(Error::UnsupportedRevision); + } else if mode.contains(RevparseMode::RANGE) { + exclude.push(commit(spec.from())?); + include.push(commit(spec.to())?); + } else { + include.push(commit(spec.from())?); } } - Ok(()) -} - -fn resolve<'repo>(repo: &'repo Repository, revision: &str) -> Result, Error> { - repo.revparse_single(revision) - .map_err(Error::ParseRevision)? - .peel_to_commit() - .map_err(Error::PeelToCommit) + Ok(Revisions { include, exclude }) } -fn peel(object: &Object<'_>) -> Result { - object - .peel_to_commit() - .map(|commit| commit.id()) - .map_err(Error::PeelToCommit) +// The selected commits, parents first. +fn walk(repo: &Repository, include: &[Oid], exclude: &[Oid]) -> Result, Error> { + let mut walk = repo.revwalk().map_err(Error::WalkHistory)?; + walk.set_sorting(Sort::TOPOLOGICAL | Sort::REVERSE) + .map_err(Error::WalkHistory)?; + for id in include { + walk.push(*id).map_err(Error::WalkHistory)?; + } + for id in exclude { + walk.hide(*id).map_err(Error::WalkHistory)?; + } + walk.map(|id| id.map_err(Error::WalkHistory)).collect() } #[cfg(test)] mod tests { use super::{Oid, Repository, resolve_commits}; + use crate::Error; use tempfile::TempDir; struct History { @@ -162,8 +122,8 @@ mod tests { .unwrap() } - fn select(fixture: &History, revisions: &[&str], all: bool) -> Result, crate::Error> { - resolve_commits(&fixture.repository, revisions, all) + fn select(fixture: &History, revisions: &[&str]) -> Result, Error> { + resolve_commits(&fixture.repository, revisions) .map(|commits| commits.into_iter().map(|commit| commit.0).collect()) } @@ -179,9 +139,10 @@ mod tests { let fixture = History::new(); let base = fixture.commits[3]; let tree = fixture.repository.find_commit(base).unwrap().tree_id(); + let repository = &fixture.repository; let commit = |message, parents: &[Oid], timestamp| { commit_as( - &fixture.repository, + repository, None, message, tree, @@ -198,16 +159,15 @@ mod tests { let b = commit("B", &[a], 100); let head = commit("Merge", &[b, fork], 100); - let commits = select(&fixture, &[&format!("{base}..{head}")], false).unwrap(); + let commits = select(&fixture, &[&format!("{base}..{head}")]).unwrap(); assert_eq!(commits, vec![fork, left, a, b, head]); } #[test] - fn combines_multiple_revisions_ranges_and_exclusions() { + fn combines_multiple_revisions_and_ranges() { // History: Base -> First -> Second -> Third (HEAD), plus a "side" - // branch that also starts from Base. Each case passes several - // arguments at once, like `git log`, and checks what they select - // together. + // branch that also starts from Base. Several arguments select the + // union of what each one selects, like `git log`. let fixture = History::new(); let base = fixture.commits[0]; let tree = fixture.repository.find_commit(base).unwrap().tree_id(); @@ -220,96 +180,29 @@ mod tests { .repository .reference("refs/heads/base", base, false, "test") .unwrap(); - let cases = [ - // "^base" leaves out Base and everything before it. - ( - vec!["HEAD", "side", "^base"], - vec![ - fixture.commits[1], - fixture.commits[2], - fixture.commits[3], - side, - ], - ), - // "base..X" means "commits in X that are not in base". - ( - vec!["base..HEAD", "base..side"], - vec![ - fixture.commits[1], - fixture.commits[2], - fixture.commits[3], - side, - ], - ), - // "X^@" means "X's parents and their history", without X itself. - (vec!["HEAD^@", "^base"], fixture.commits[1..3].to_vec()), - // "X^!" means "only commit X". - (vec!["HEAD^!", "side^!"], vec![fixture.commits[3], side]), - ]; - for (args, expected) in cases { - let commits = select(&fixture, &args, false).unwrap(); - assert_selection(commits, &expected); - } - } - - #[test] - fn all_includes_branches_tags_and_detached_head() { - // --all must select every commit reachable from a branch, a tag, or - // HEAD. HEAD is detached here, so no branch points to "Detached"; - // it can only be found through HEAD. - let fixture = History::new(); - let base = fixture.commits[0]; - let tree = fixture.repository.find_commit(base).unwrap().tree_id(); - let side = commit_as(&fixture.repository, None, "Side", tree, [base], 100); - fixture - .repository - .reference("refs/heads/side", side, false, "test") - .unwrap(); - let tagged = commit_as(&fixture.repository, None, "Tagged", tree, [base], 200); - fixture - .repository - .tag( - "release", - &fixture.repository.find_object(tagged, None).unwrap(), - &signature(200), - "Release", - false, - ) - .unwrap(); - let detached = commit_as(&fixture.repository, None, "Detached", tree, [base], 300); - fixture.repository.set_head_detached(detached).unwrap(); - let mut expected = fixture.commits.clone(); - expected.extend([side, tagged, detached]); - let commits = select(&fixture, &[], true).unwrap(); + let commits = select(&fixture, &["base..HEAD", "base..side"]).unwrap(); + let mut expected = fixture.commits[1..].to_vec(); + expected.push(side); assert_selection(commits, &expected); } #[test] - fn symmetric_ranges_hide_all_merge_bases_and_allow_unrelated_histories() { - // "left...right" means "commits on either side, but not on both". - // Left and Right are both merges of A and B, so they share two - // ancestors. Both must be left out, leaving only Left and Right. + fn rejects_revisions_beyond_x_and_ranges() { + // "A...B" needs common ancestors, which a shallow clone may not + // have, so it is rejected in every clone. libgit2 does not parse + // "^X", "X^!" or "X^@" at all. let fixture = History::new(); - let base = fixture.commits[0]; - let tree = fixture.repository.find_commit(base).unwrap().tree_id(); - let a = commit_as(&fixture.repository, None, "A", tree, [base], 100); - let b = commit_as(&fixture.repository, None, "B", tree, [base], 900); - let left = commit_as(&fixture.repository, None, "Left", tree, [a, b], 10); - let right = commit_as(&fixture.repository, None, "Right", tree, [b, a], 20); - // Make sure the setup really has two shared ancestors. - assert_eq!( - fixture.repository.merge_bases(left, right).unwrap().len(), - 2 + let result = select(&fixture, &["HEAD~2...HEAD"]); + assert!( + matches!(result, Err(Error::UnsupportedRevision)), + "{result:?}" ); - let commits = select(&fixture, &[&format!("{left}...{right}")], false).unwrap(); - assert_selection(commits, &[left, right]); - - // Unrelated shares no history with HEAD. Instead of failing, the - // range should select every commit from both sides. - let unrelated = commit_as(&fixture.repository, None, "Unrelated", tree, [], 5); - let commits = select(&fixture, &[&format!("HEAD...{unrelated}")], false).unwrap(); - let mut expected = fixture.commits.clone(); - expected.push(unrelated); - assert_selection(commits, &expected); + for revision in ["^HEAD", "HEAD^!", "HEAD^@"] { + let result = select(&fixture, &[revision]); + assert!( + matches!(result, Err(Error::ParseRevision(_))), + "{revision}: {result:?}" + ); + } } } diff --git a/src/lib.rs b/src/lib.rs index 0d48bba..f4eb94c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,9 @@ use std::io::Write; use std::path::PathBuf; use std::process::ExitStatus; -pub fn run_revisions(revisions: &[&str], all: bool, output: &mut impl Write) -> Result<(), Error> { +pub fn run_revisions(revisions: &[&str], output: &mut impl Write) -> Result<(), Error> { let repository = discover_repository()?; - let commits = history::resolve_commits(&repository.git_repo, revisions, all)?; + let commits = history::resolve_commits(&repository.git_repo, revisions)?; if commits.is_empty() { writeln!(output, "No commits in range.").map_err(Error::WriteOutput)?; @@ -58,14 +58,12 @@ pub enum Error { CanonicalizeRepository(#[source] std::io::Error), #[error("could not parse the revision: {0}")] ParseRevision(#[source] git2::Error), - #[error("unsupported revision mode: {0:?}")] - UnsupportedRevisionMode(git2::RevparseMode), + #[error("only X and A..B revisions are supported")] + UnsupportedRevision, #[error("could not resolve a selected commit: {0}")] PeelToCommit(#[source] git2::Error), #[error("could not walk the selected history: {0}")] WalkHistory(#[source] git2::Error), - #[error("could not find common ancestors: {0}")] - MergeBases(#[source] git2::Error), #[error("could not write output: {0}")] WriteOutput(#[source] std::io::Error), #[error("could not start Nix while checking commit {commit}: {source}")] diff --git a/src/main.rs b/src/main.rs index 5e582ab..5ea2029 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,18 +5,16 @@ use std::process::ExitCode; #[derive(Parser)] #[command(version, about)] struct Cli { - /// Git revisions or ranges to include; prefix a revision with ^ to exclude it - #[arg(value_name = "REVISION", required_unless_present = "all")] + /// Revisions to check: X includes X and its history, and A..B means B + /// but not A + #[arg(value_name = "REVISION", required = true)] revisions: Vec, - /// Include history reachable from all Git refs and HEAD - #[arg(long)] - all: bool, } fn main() -> ExitCode { let cli = Cli::parse(); let revisions: Vec<_> = cli.revisions.iter().map(String::as_str).collect(); - match validate_commits::run_revisions(&revisions, cli.all, &mut io::stdout()) { + match validate_commits::run_revisions(&revisions, &mut io::stdout()) { Ok(()) => ExitCode::SUCCESS, Err(error) => { eprintln!("error: {error}"); diff --git a/tests/cli.rs b/tests/cli.rs index 003f160..7dbfc98 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -165,7 +165,7 @@ fn stops_when_a_selected_commits_check_fails() { let passing = fixture.commit("Passing commit"); let failing = fixture.flake_commit("Failing commit", FAILING); fixture.flake_commit("Passing tip", PASSING); - let output = fixture.run_args(&["--all", &format!("^{base}")]); + let output = fixture.run(&format!("{base}..HEAD")); assert!(!output.status.success()); assert_eq!(validated(&output), [passing, failing]); let error = String::from_utf8_lossy(&output.stderr); @@ -188,7 +188,7 @@ fn checks_a_commit_at_a_shallow_boundary() { fs::remove_file(objects.join(&parent[..2]).join(&parent[2..])).unwrap(); let shallow = fixture.repository.commondir().join("shallow"); fs::write(&shallow, format!("{head}\n")).unwrap(); - let output = fixture.run("HEAD^!"); + let output = fixture.run("HEAD"); assert!( output.status.success(), "{}", From b70684004f6b66f75e23e9b5335342cefc545e34 Mon Sep 17 00:00:00 2001 From: Mshehu5 Date: Sat, 14 Nov 2026 12:45:17 +0100 Subject: [PATCH 3/3] Fetch missing history in shallow clones CI checks out a shallow clone, so the base of BASE..HEAD is often missing. Fetch a missing base by its ID, then deepen from the points where history stops (8, 16, 32 commits...) until the selection can't change. Stop with an error if origin can't provide more history. libgit2 treats the commits where a shallow clone stops as roots, so its walk and topological sort work unchanged. It has no API for those commits or their real parents, so they come from the shallow file and the raw commit. Fetching uses git: libgit2 can't fetch shallowly from a local repository and ignores http.extraheader, which GitHub's checkout uses to log in. libgit2 caches commits and reads the shallow file once, so the repository is reopened after each fetch. --- nix/package.nix | 12 +- src/history.rs | 300 ++++++++++++++++++++++++++++++++++++++------ src/lib.rs | 21 +++- src/shallow.rs | 117 +++++++++++++++++ src/test_support.rs | 112 +++++++++++++++++ tests/cli.rs | 47 +++++-- 6 files changed, 557 insertions(+), 52 deletions(-) create mode 100644 src/shallow.rs create mode 100644 src/test_support.rs diff --git a/nix/package.nix b/nix/package.nix index 27dfa79..ce24994 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -17,7 +17,10 @@ strictDeps = true; nativeBuildInputs = [ pkgs.pkg-config ]; buildInputs = [ pkgs.zlib ]; - nativeCheckInputs = [ pkgs.nix ]; + nativeCheckInputs = [ + pkgs.nix + pkgs.git + ]; }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; validate-commits = craneLib.buildPackage ( @@ -27,7 +30,12 @@ nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ pkgs.makeWrapper ]; postInstall = '' wrapProgram "$out/bin/validate-commits" \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix ]} + --prefix PATH : ${ + pkgs.lib.makeBinPath [ + pkgs.nix + pkgs.git + ] + } ''; } ); diff --git a/src/history.rs b/src/history.rs index 30cb57a..efb96b0 100644 --- a/src/history.rs +++ b/src/history.rs @@ -1,16 +1,59 @@ use git2::{Oid, Repository, RevparseMode, Sort}; +use std::collections::HashSet; +use std::io::Write; +use crate::shallow::{cutoffs, fetch, missing_commit}; use crate::{CommitId, Error}; // Find the commits to validate: reachable from an included revision but -// not from an excluded one, with parents before their children. +// not from an excluded one, with parents before their children. In a +// shallow clone, fetch more history until the answer cannot change. pub(crate) fn resolve_commits( - repo: &Repository, + repo: &mut Repository, revisions: &[&str], + output: &mut impl Write, ) -> Result, Error> { - let parsed = parse_revisions(repo, revisions)?; - let selected = walk(repo, &parsed.include, &parsed.exclude)?; - Ok(selected.into_iter().map(CommitId).collect()) + let mut deepen = 8u32; + loop { + // A revision given as a full commit ID may not be in the clone yet, + // like the base commit CI passes in. Fetch just those commits first. + let mut missing: Vec<_> = revisions + .iter() + .flat_map(|input| input.split("..")) + .filter_map(|name| missing_commit(repo, name)) + .collect(); + missing.sort(); + missing.dedup(); + if !missing.is_empty() { + fetch(repo, &missing, None, output)?; + continue; + } + let parsed = parse_revisions(repo, revisions)?; + let selected = walk(repo, &parsed.include, &parsed.exclude)?; + if !repo.is_shallow() { + return Ok(selected.into_iter().map(CommitId).collect()); + } + // Only cut-off points behind a revision can change the selection. + let behind: HashSet<_> = cutoffs(repo)? + .into_iter() + .filter(|cutoff| { + parsed + .include + .iter() + .chain(&parsed.exclude) + .any(|id| descends(repo, *id, *cutoff)) + }) + .collect(); + if complete(repo, &parsed, &selected, &behind) { + return Ok(selected.into_iter().map(CommitId).collect()); + } + // Deepen from the cut-off points, not the revisions: a local commit + // may not exist on origin. + let mut behind: Vec<_> = behind.into_iter().collect(); + behind.sort(); + fetch(repo, &behind, Some(deepen), output)?; + deepen = deepen.saturating_mul(2); + } } // The commits named by the arguments, split into included and excluded. @@ -47,7 +90,8 @@ fn parse_revisions(repo: &Repository, revisions: &[&str]) -> Result Result, Error> { let mut walk = repo.revwalk().map_err(Error::WalkHistory)?; walk.set_sorting(Sort::TOPOLOGICAL | Sort::REVERSE) @@ -61,10 +105,58 @@ fn walk(repo: &Repository, include: &[Oid], exclude: &[Oid]) -> Result, walk.map(|id| id.map_err(Error::WalkHistory)).collect() } +// Whether `ancestor` is `id` or reachable from it in the downloaded history. +fn descends(repo: &Repository, id: Oid, ancestor: Oid) -> bool { + id == ancestor || repo.graph_descendant_of(id, ancestor).unwrap_or(false) +} + +// Check whether missing history could still change the selected commits. +fn complete( + repo: &Repository, + revisions: &Revisions, + selected: &[Oid], + cutoffs: &HashSet, +) -> bool { + if cutoffs.is_empty() || selected.is_empty() { + return true; + } + // A selected commit's parents are missing, so older selected commits + // may be missing too. + if selected.iter().any(|id| cutoffs.contains(id)) { + return false; + } + let excluded_cutoffs: Vec<_> = cutoffs + .iter() + .filter(|cutoff| { + revisions + .exclude + .iter() + .any(|id| descends(repo, *id, **cutoff)) + }) + .collect(); + // Missing excluded history could still exclude an apparent first + // selected commit, one whose parents are all excluded. It is safe only + // when that commit already descends from every excluded cut-off point. + let chosen: HashSet<_> = selected.iter().collect(); + selected.iter().all(|id| { + let first = repo + .find_commit(*id) + .map(|commit| commit.parent_ids().all(|parent| !chosen.contains(&parent))) + .unwrap_or(true); + !first + || excluded_cutoffs + .iter() + .all(|cutoff| descends(repo, *id, **cutoff)) + }) +} + #[cfg(test)] mod tests { - use super::{Oid, Repository, resolve_commits}; + use super::resolve_commits; use crate::Error; + use crate::test_support::{Remote, commit_as, git}; + use git2::{Oid, Repository}; + use std::fs; use tempfile::TempDir; struct History { @@ -98,32 +190,12 @@ mod tests { } } - fn signature(timestamp: i64) -> git2::Signature<'static> { - git2::Signature::new("Test", "test@example.com", &git2::Time::new(timestamp, 0)).unwrap() + fn select(fixture: &mut History, revisions: &[&str]) -> Result, Error> { + select_in(&mut fixture.repository, revisions) } - fn commit_as( - repository: &Repository, - reference: Option<&str>, - message: &str, - tree: Oid, - parents: impl IntoIterator, - timestamp: i64, - ) -> Oid { - let signature = signature(timestamp); - let tree = repository.find_tree(tree).unwrap(); - let parents: Vec<_> = parents - .into_iter() - .map(|id| repository.find_commit(id).unwrap()) - .collect(); - let parents: Vec<_> = parents.iter().collect(); - repository - .commit(reference, &signature, &signature, message, &tree, &parents) - .unwrap() - } - - fn select(fixture: &History, revisions: &[&str]) -> Result, Error> { - resolve_commits(&fixture.repository, revisions) + fn select_in(repository: &mut Repository, revisions: &[&str]) -> Result, Error> { + resolve_commits(repository, revisions, &mut Vec::new()) .map(|commits| commits.into_iter().map(|commit| commit.0).collect()) } @@ -136,7 +208,7 @@ mod tests { #[test] fn orders_merge_parents_with_skewed_dates() { - let fixture = History::new(); + let mut fixture = History::new(); let base = fixture.commits[3]; let tree = fixture.repository.find_commit(base).unwrap().tree_id(); let repository = &fixture.repository; @@ -159,7 +231,7 @@ mod tests { let b = commit("B", &[a], 100); let head = commit("Merge", &[b, fork], 100); - let commits = select(&fixture, &[&format!("{base}..{head}")]).unwrap(); + let commits = select(&mut fixture, &[&format!("{base}..{head}")]).unwrap(); assert_eq!(commits, vec![fork, left, a, b, head]); } @@ -168,7 +240,7 @@ mod tests { // History: Base -> First -> Second -> Third (HEAD), plus a "side" // branch that also starts from Base. Several arguments select the // union of what each one selects, like `git log`. - let fixture = History::new(); + let mut fixture = History::new(); let base = fixture.commits[0]; let tree = fixture.repository.find_commit(base).unwrap().tree_id(); let side = commit_as(&fixture.repository, None, "Side", tree, [base], 100); @@ -180,7 +252,7 @@ mod tests { .repository .reference("refs/heads/base", base, false, "test") .unwrap(); - let commits = select(&fixture, &["base..HEAD", "base..side"]).unwrap(); + let commits = select(&mut fixture, &["base..HEAD", "base..side"]).unwrap(); let mut expected = fixture.commits[1..].to_vec(); expected.push(side); assert_selection(commits, &expected); @@ -191,18 +263,170 @@ mod tests { // "A...B" needs common ancestors, which a shallow clone may not // have, so it is rejected in every clone. libgit2 does not parse // "^X", "X^!" or "X^@" at all. - let fixture = History::new(); - let result = select(&fixture, &["HEAD~2...HEAD"]); + let mut fixture = History::new(); + let result = select(&mut fixture, &["HEAD~2...HEAD"]); assert!( matches!(result, Err(Error::UnsupportedRevision)), "{result:?}" ); for revision in ["^HEAD", "HEAD^!", "HEAD^@"] { - let result = select(&fixture, &[revision]); + let result = select(&mut fixture, &[revision]); assert!( matches!(result, Err(Error::ParseRevision(_))), "{revision}: {result:?}" ); } } + + // Shallow clone tests: history is built in a "remote" repository, then + // cloned with only a few commits, the way CI checks out a pull request. + // The tool must fetch just enough missing history to pick the right + // commits. + #[test] + fn shallow_clones_fetch_enough_history_to_select_a_range() { + // A pull request of three commits merged on top of a long main + // line. At depth 1 the base commit is missing entirely; at depth 2 + // it is present but the PR's history is cut off. Either way, the + // tool must fetch more history and then select exactly the PR. + for depth in [1, 2] { + let remote = Remote::new(); + let base = *remote.chain(None, 20, "Main").last().unwrap(); + let mut selected = remote.chain(Some(base), 3, "PR"); + let head = remote.merge([base, *selected.last().unwrap()]); + selected.push(head); + let mut clone = remote.clone_shallow(head, depth); + fs::write(clone.path().join("FETCH_HEAD"), "keep me\n").unwrap(); + + let commits = select_in(&mut clone, &[&format!("{base}..HEAD")]).unwrap(); + assert_selection(commits, &selected); + // Fetching must not overwrite FETCH_HEAD, and should not + // download the whole history. + assert_eq!( + fs::read_to_string(clone.path().join("FETCH_HEAD")).unwrap(), + "keep me\n" + ); + assert!(clone.is_shallow()); + } + } + + #[test] + fn shallow_clones_fetch_base_history_to_exclude_an_old_fork_point() { + // The PR branched from Fork, which is old history of the base + // (ten commits back). Before enough base history is fetched, Fork + // looks like part of the PR. The tool must keep fetching until it + // can see that Fork is already in the base, and select only PR and + // the merge. + let remote = Remote::new(); + let shared = *remote.chain(None, 3, "Shared").last().unwrap(); + let fork = *remote.chain(Some(shared), 1, "Fork").last().unwrap(); + let branch = *remote.chain(Some(shared), 1, "Branch").last().unwrap(); + let main = *remote.chain(Some(fork), 10, "Main").last().unwrap(); + let base = remote.merge([main, branch]); + let pr = *remote.chain(Some(fork), 1, "PR").last().unwrap(); + let head = remote.merge([base, pr]); + let mut clone = remote.clone_shallow(head, 1); + + let commits = select_in(&mut clone, &[&format!("{base}..HEAD")]).unwrap(); + assert_selection(commits, &[pr, head]); + } + + #[test] + fn shallow_clones_fetch_history_for_local_commits_origin_has_not_seen() { + // A developer with a shallow clone adds a commit that is not pushed + // yet. Origin cannot send history for that commit by its ID, so the + // tool must ask for history of the cut-off commits origin does know. + let remote = Remote::new(); + let base = *remote.chain(None, 20, "Main").last().unwrap(); + let mut selected = remote.chain(Some(base), 3, "PR"); + let mut clone = remote.clone_shallow(*selected.last().unwrap(), 2); + let head = clone.head().unwrap().peel_to_commit().unwrap(); + let local = commit_as( + &clone, + Some("HEAD"), + "Local", + head.tree_id(), + [head.id()], + 0, + ); + drop(head); + selected.push(local); + + let commits = select_in(&mut clone, &[&format!("{base}..HEAD")]).unwrap(); + assert_selection(commits, &selected); + } + + #[test] + fn shallow_clones_fetch_every_missing_range_end() { + // Two ranges whose ends are both cut off from a depth-1 clone. Each + // end is fetched by its ID, and together they select both branches. + let remote = Remote::new(); + let base = *remote.chain(None, 20, "Main").last().unwrap(); + let pr = remote.chain(Some(base), 3, "PR"); + let side = remote.chain(Some(base), 2, "Side"); + let (pr_tip, side_tip) = (*pr.last().unwrap(), *side.last().unwrap()); + let head = remote.merge([pr_tip, side_tip]); + let mut clone = remote.clone_shallow(head, 1); + + let ranges = [format!("{base}..{pr_tip}"), format!("{base}..{side_tip}")]; + let commits = select_in(&mut clone, &[&ranges[0], &ranges[1]]).unwrap(); + assert_selection(commits, &[pr, side].concat()); + } + + #[test] + fn shallow_clones_report_fetch_failures() { + // If history cannot be fetched, the tool must stop with an error + // instead of checking a wrong set of commits. + let remote = Remote::new(); + let base = *remote.chain(None, 20, "Main").last().unwrap(); + let head = *remote.chain(Some(base), 3, "PR").last().unwrap(); + let mut clone = remote.clone_shallow(head, 1); + git( + clone.workdir().unwrap(), + &[ + "remote", + "set-url", + "origin", + "/nonexistent/validate-commits", + ], + ); + + let result = select_in(&mut clone, &[&format!("{base}..HEAD")]); + assert!( + matches!(result, Err(Error::FetchFailed { .. })), + "{result:?}" + ); + } + + #[test] + fn shallow_clones_stop_when_origin_is_also_shallow() { + // Origin is itself shallow: it has the newest commit and the base, + // but not the history between them. The tool must notice that + // fetching made no progress and stop with a clear error. + let remote = Remote::new(); + let commits = remote.chain(None, 20, "Main"); + let (base, head) = (commits[10], *commits.last().unwrap()); + let origin = remote.clone_shallow(head, 1); + let origin_dir = origin.workdir().unwrap().to_owned(); + git( + &origin_dir, + &["fetch", "--quiet", "--depth=1", "origin", &base.to_string()], + ); + git( + &origin_dir, + &["config", "uploadpack.allowAnySHA1InWant", "true"], + ); + let root = tempfile::tempdir().unwrap(); + let url = format!("file://{}", origin_dir.display()); + git( + root.path(), + &["clone", "--quiet", "--depth=1", &url, "clone"], + ); + let mut clone = Repository::open(root.path().join("clone")).unwrap(); + + let result = select_in(&mut clone, &[&format!("{base}..HEAD")]); + assert!( + matches!(result, Err(Error::NoHistoryProgress)), + "{result:?}" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index f4eb94c..5339c29 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,8 @@ mod history; mod nix; +mod shallow; +#[cfg(test)] +mod test_support; use std::fmt; use std::io::Write; @@ -7,8 +10,8 @@ use std::path::PathBuf; use std::process::ExitStatus; pub fn run_revisions(revisions: &[&str], output: &mut impl Write) -> Result<(), Error> { - let repository = discover_repository()?; - let commits = history::resolve_commits(&repository.git_repo, revisions)?; + let mut repository = discover_repository()?; + let commits = history::resolve_commits(&mut repository.git_repo, revisions, output)?; if commits.is_empty() { writeln!(output, "No commits in range.").map_err(Error::WriteOutput)?; @@ -64,6 +67,20 @@ pub enum Error { PeelToCommit(#[source] git2::Error), #[error("could not walk the selected history: {0}")] WalkHistory(#[source] git2::Error), + #[error("could not read shallow history: {0}")] + ReadShallow(#[source] std::io::Error), + #[error("could not read a commit: {0}")] + FindCommit(#[source] git2::Error), + #[error("could not run git fetch: {0}")] + SpawnGit(#[source] std::io::Error), + #[error("git fetch failed{}", status_suffix(.status))] + FetchFailed { status: ExitStatus }, + #[error("could not reopen the repository after fetching: {0}")] + ReopenRepository(#[source] git2::Error), + #[error("fetch did not supply requested commit {0}")] + MissingFetchedCommit(git2::Oid), + #[error("fetch did not deepen history; origin may itself be shallow")] + NoHistoryProgress, #[error("could not write output: {0}")] WriteOutput(#[source] std::io::Error), #[error("could not start Nix while checking commit {commit}: {source}")] diff --git a/src/shallow.rs b/src/shallow.rs new file mode 100644 index 0000000..cc2718c --- /dev/null +++ b/src/shallow.rs @@ -0,0 +1,117 @@ +// Shallow clones: find where downloaded history stops and fetch more of it. +// +// libgit2 reads the clone's `shallow` file as grafts, so a commit where +// history stops looks like it has no parents. That keeps its history walk +// working, but it has no API that lists those commits or their real +// parents, so both are read here. + +use git2::{Commit, Oid, Repository}; +use std::collections::HashSet; +use std::io::Write; +use std::process::{Command, Stdio}; + +use crate::Error; + +// Return the commit a full commit ID names if this clone does not have it +// yet, so it can be fetched. Abbreviated IDs and expressions such as SHA~1 +// are never fetched. +pub(crate) fn missing_commit(repo: &Repository, revision: &str) -> Option { + if revision.len() != 40 { + return None; + } + let id = Oid::from_str(revision).ok()?; + let odb = repo.odb().ok()?; + (!odb.exists(id)).then_some(id) +} + +// A commit's parents as recorded in the commit itself. libgit2's +// `parent_ids` leaves out parents a shallow clone cut off. +pub(crate) fn real_parents(commit: &Commit<'_>) -> Vec { + commit + .raw_header_bytes() + .split(|byte| *byte == b'\n') + .filter_map(|line| line.strip_prefix(b"parent ")) + .filter_map(|id| std::str::from_utf8(id).ok()) + .filter_map(|id| Oid::from_str(id).ok()) + .collect() +} + +// The commits where this clone's downloaded history stops, from the +// repository's `shallow` file. Linked worktrees share it. +fn boundaries(repo: &Repository) -> Result, Error> { + match std::fs::read_to_string(repo.commondir().join("shallow")) { + Ok(contents) => Ok(contents + .lines() + .filter_map(|line| Oid::from_str(line).ok()) + .collect()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(Error::ReadShallow(error)), + } +} + +// The boundary commits whose parents exist on origin but not here. Real +// root commits have no parents to fetch. +pub(crate) fn cutoffs(repo: &Repository) -> Result, Error> { + let mut cutoffs = HashSet::new(); + for id in boundaries(repo)? { + let commit = repo.find_commit(id).map_err(Error::FindCommit)?; + if !real_parents(&commit).is_empty() { + cutoffs.insert(id); + } + } + Ok(cutoffs) +} + +// Fetch commits or more ancestry without changing branches or staged files, +// then reopen the repository: libgit2 reads the `shallow` file and caches +// commits when it opens a repository, so it would not see the new history. +// +// This runs `git fetch` rather than libgit2's fetch. libgit2 refuses +// shallow fetches from local repositories, and it ignores the +// `http.extraheader` setting that GitHub's checkout action uses to log in. +pub(crate) fn fetch( + repo: &mut Repository, + tips: &[Oid], + deepen: Option, + output: &mut impl Write, +) -> Result<(), Error> { + let before = boundaries(repo)?; + let depth_flag = match deepen { + Some(count) => { + writeln!(output, "Deepening history by {count} commits").map_err(Error::WriteOutput)?; + output.flush().map_err(Error::WriteOutput)?; + format!("--deepen={count}") + } + None => "--depth=1".to_owned(), + }; + let status = Command::new("git") + .current_dir(repo.workdir().ok_or(Error::BareRepository)?) + .args([ + "fetch", + "--no-tags", + "--no-write-fetch-head", + "--refmap=", + "--no-auto-maintenance", + "--no-recurse-submodules", + ]) + .arg(depth_flag) + .arg("origin") + .args(tips.iter().map(ToString::to_string)) + .stdout(Stdio::null()) + .status() + .map_err(Error::SpawnGit)?; + if !status.success() { + return Err(Error::FetchFailed { status }); + } + *repo = Repository::open(repo.path()).map_err(Error::ReopenRepository)?; + if deepen.is_some() && boundaries(repo)? == before { + return Err(Error::NoHistoryProgress); + } + let odb = repo.odb().map_err(Error::ReopenRepository)?; + for id in tips { + if !odb.exists(*id) { + return Err(Error::MissingFetchedCommit(*id)); + } + } + Ok(()) +} diff --git a/src/test_support.rs b/src/test_support.rs new file mode 100644 index 0000000..58c6358 --- /dev/null +++ b/src/test_support.rs @@ -0,0 +1,112 @@ +// Repositories for tests, built with git2. + +use git2::{Oid, Repository, Signature, Time}; +use std::path::Path; +use std::process::Command; +use tempfile::TempDir; + +// Write a commit with a fixed date. With a reference, also point it (or +// HEAD's branch) at the new commit. +pub(crate) fn commit_as( + repository: &Repository, + reference: Option<&str>, + message: &str, + tree: Oid, + parents: impl IntoIterator, + timestamp: i64, +) -> Oid { + let signature = Signature::new("Test", "test@example.com", &Time::new(timestamp, 0)).unwrap(); + let tree = repository.find_tree(tree).unwrap(); + let parents: Vec<_> = parents + .into_iter() + .map(|id| repository.find_commit(id).unwrap()) + .collect(); + let parents: Vec<_> = parents.iter().collect(); + repository + .commit(reference, &signature, &signature, message, &tree, &parents) + .unwrap() +} + +// History built in a "remote" repository, then cloned with only a few +// commits, the way CI checks out a pull request. +pub(crate) struct Remote { + root: TempDir, + pub(crate) repository: Repository, + tree: Oid, +} + +impl Remote { + pub(crate) fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let repository = Repository::init(root.path().join("remote")).unwrap(); + // Let clones fetch any commit by its ID, as GitHub allows. + repository + .config() + .unwrap() + .set_bool("uploadpack.allowAnySHA1InWant", true) + .unwrap(); + let tree = repository.treebuilder(None).unwrap().write().unwrap(); + Self { + root, + repository, + tree, + } + } + + // Add `count` commits on top of `parent`, oldest first. + pub(crate) fn chain(&self, parent: Option, count: usize, name: &str) -> Vec { + let mut commits: Vec = Vec::new(); + for i in 0..count { + let parent = commits.last().copied().or(parent); + let message = format!("{name} {i}"); + commits.push(commit_as( + &self.repository, + None, + &message, + self.tree, + parent, + i as i64, + )); + } + commits + } + + pub(crate) fn merge(&self, parents: [Oid; 2]) -> Oid { + commit_as(&self.repository, None, "Merge", self.tree, parents, 0) + } + + // Clone only the newest `depth` commits of `tip`. libgit2 cannot make + // shallow clones of local repositories, so this uses `git clone`. + pub(crate) fn clone_shallow(&self, tip: Oid, depth: usize) -> Repository { + self.repository + .reference("refs/heads/queue", tip, true, "test") + .unwrap(); + let url = format!("file://{}", self.repository.path().display()); + git( + self.root.path(), + &[ + "clone", + "--quiet", + &format!("--depth={depth}"), + "--branch=queue", + &url, + "clone", + ], + ); + Repository::open(self.root.path().join("clone")).unwrap() + } +} + +pub(crate) fn git(directory: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(args) + .current_dir(directory) + .output() + .expect("git must be on PATH to run the shallow clone tests"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} diff --git a/tests/cli.rs b/tests/cli.rs index 7dbfc98..20b71d9 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -124,6 +124,19 @@ impl Fixture { } } +fn git(directory: &std::path::Path, args: &[&str]) { + let output = Command::new("git") + .args(args) + .current_dir(directory) + .output() + .expect("git must be on PATH to run the shallow clone test"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + fn validated(output: &Output) -> Vec { String::from_utf8_lossy(&output.stdout) .lines() @@ -177,22 +190,36 @@ fn stops_when_a_selected_commits_check_fails() { } #[test] -fn checks_a_commit_at_a_shallow_boundary() { - // Mimic a depth-1 clone: HEAD is a shallow boundary whose parent - // object is missing from the repository. +fn checks_commits_in_a_shallow_clone() { + // Like a CI checkout: clone only the newest commit, then validate a + // range whose base is not in the clone. The tool must fetch enough + // history, and must tell Nix the clone is shallow or Nix refuses it. let mut fixture = Fixture::new(); - let parent = fixture.flake_commit("Parent", PASSING); + fixture.flake_commit("Flake", PASSING); + for i in 0..10 { + fixture.commit(&format!("Old {i}")); + } + let base = fixture.commit("Base"); let head = fixture.commit("Head"); - let objects = fixture.repository.path().join("objects"); - let parent = parent.to_string(); - fs::remove_file(objects.join(&parent[..2]).join(&parent[2..])).unwrap(); - let shallow = fixture.repository.commondir().join("shallow"); - fs::write(&shallow, format!("{head}\n")).unwrap(); - let output = fixture.run("HEAD"); + // Let the clone fetch any commit by its ID, as GitHub allows. + git( + &fixture.repo, + &["config", "uploadpack.allowAnySHA1InWant", "true"], + ); + let url = format!("file://{}", fixture.repo.display()); + git( + fixture.root.path(), + &["clone", "--quiet", "--depth=1", &url, "clone"], + ); + fixture.repo = fixture.root.path().join("clone"); + fixture.repository = Repository::open(&fixture.repo).unwrap(); + + let output = fixture.run(&format!("{base}..HEAD")); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(validated(&output), [head]); + assert!(Repository::open(&fixture.repo).unwrap().is_shallow()); }