Skip to content

Move some git calls to libgit #407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
seanses wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Move some git calls to libgit #407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6a774b4
write repo fetch config
seanses Jul 1, 2024
cb9f929
show repo config file
seanses Jul 2, 2024
76bf20c
sync note refs to local, drop git show-ref call
seanses Jul 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions rust/gitxetcore/src/config/xet.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ use crate::constants::{
use crate::data::remote_shard_interface::{GlobalDedupPolicy, SmudgeQueryPolicy};
use crate::errors::GitXetRepoError;
use crate::git_integration::git_url::ssh_url_to_https_url;
use crate::git_integration::{run_git_captured, GitXetRepo};
use crate::git_integration::read_file_from_repo;
use crate::git_integration::GitXetRepo;
use crate::xetblob::get_cas_endpoint_from_git_remote;
use lazy_static::lazy_static;
use parutils::tokio_par_for_any_ok;
Expand Down Expand Up @@ -629,22 +630,20 @@ impl XetConfig {
}

fn try_with_repo_config_file(mut self, repo_dir: &PathBuf) -> Result<Self, ConfigError> {
let query_spec = format!("HEAD:{GIT_REPO_SPECIFIC_CONFIG}");

let Ok((status, stdout, _stderr)) =
run_git_captured(Some(repo_dir), "show", &[&query_spec], false, None)
else {
let Ok(repo) = git2::Repository::discover(repo_dir) else {
return Ok(self);
};

if status != Some(0) {
let Ok(Some(repo_xet_config)) =
read_file_from_repo(&Arc::new(repo), GIT_REPO_SPECIFIC_CONFIG, None)
else {
return Ok(self);
}
};

if let Ok(local_config) = toml::from_str::<LocalXetRepoConfig>(&stdout).map_err(
if let Ok(local_config) = toml::from_slice::<LocalXetRepoConfig>(&repo_xet_config).map_err(
|e|
{
let msg = format!("Warning: Error parsing local config ref {query_spec}: {e:?}. Please correct the errors and commit the corrected version into the repo.");
let msg = format!("Warning: Error parsing local config ref {GIT_REPO_SPECIFIC_CONFIG}: {e:?}. Please correct the errors and commit the corrected version into the repo.");
eprintln!("{msg}");
}) {
self.upstream_xet_repo = local_config.upstream;
Expand Down
99 changes: 64 additions & 35 deletions rust/gitxetcore/src/git_integration/git_xet_repo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use mdb_shard::constants::MDB_SHARD_MIN_TARGET_SIZE;
use mdb_shard::error::MDBShardError;
use mdb_shard::session_directory::consolidate_shards_in_directory;
use mdb_shard::shard_version::ShardVersion;
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
#[cfg(unix)]
use std::fs::Permissions;
use std::fs::{self, File};
Expand Down Expand Up @@ -714,36 +714,41 @@ impl GitXetRepo {
pub async fn verify_or_write_repo_fetch_config(&self) -> Result<Vec<String>> {
let mut new_remotes: Vec<String> = Vec::new();

// Fetch the current config that matches a given pattern, saving the output.
// If we parse it, then we can determine whether things should change.

// If a new remote was added -- i.e. one of the remotes does not have a matching
// tracking note config -- then we trigger a remote fetch to pull those notes. The
// reference transaction hook should also catch it, but may not right away.
let (_, config_settings, _) = self.run_git_in_repo(
"config",
&["--get-regex", "remote\\.[a-z]+\\.fetch", ".*/notes/xet/.*"],
)?;

let repo_fetch_heads: HashMap<&str, &str> = config_settings
.split('\n')
.map(|line| line.split_once(' '))
.filter_map(|e| e.map(|vv| (vv.0.trim(), vv.1.trim())))
.collect();
let mut config: git2::Config = self.repo.config()?;

for remote in self.current_remotes()? {
let config_name = format!("remote.{}.fetch", &remote);
let config_value = format!("+refs/notes/xet/*:refs/remotes/{}/notes/xet/*", &remote);

if let Some(v) = repo_fetch_heads.get(config_name.as_str()) {
if *v == config_value {
debug!("XET: Fetch hooks on remote.{}.fetch is set.", &remote);
continue;
// Retrieve the existing fetch refspecs
let mut fetch_config_set = false;
if let Ok(mut existing_refspecs) =
config.multivar(&config_name, None /* retrieve all values */)
{
// Iterate until end (None), ignoring Errs
while let Some(maybe_refspec) = existing_refspecs.next() {
if let Ok(refspec) = maybe_refspec {
if refspec.value() == Some(&config_value) {
fetch_config_set = true;
break;
}
}
}
}

if fetch_config_set {
debug!("XET: Fetch hooks on remote.{}.fetch is set.", &remote);
continue;
}

info!("XET: Setting fetch hooks on remote.{}.fetch.", &remote);

self.run_git_checked_in_repo("config", &["--add", &config_name, &config_value])?;
config.set_multivar(&config_name, "^$" /* add a value */, &config_value)?;

new_remotes.push(remote);
}

Expand Down Expand Up @@ -1515,34 +1520,36 @@ impl GitXetRepo {

/// Syncronizes any fetched note refs to the local notes
pub fn sync_note_refs_to_local(&self, note_suffix: &str, notes_ref_suffix: &str) -> Result<()> {
let repo = self.repo.as_ref();

for xet_p in ["xet", "xet_alt"] {
let ref_suffix = format!("notes/{}/{}", &xet_p, note_suffix);

let (_, remote_refs, _) = self.run_git_in_repo("show-ref", &["--", &ref_suffix])?;

for hash_ref in remote_refs.split('\n') {
if hash_ref.is_empty() {
let references = repo.references()?;
for reference in references {
let Ok(r) = reference else {
continue;
}
};

let split_v: Vec<&str> = hash_ref.split(' ').collect();
debug_assert_eq!(split_v.len(), 2);
let remote_ref = &split_v[0];
let ref_name = &split_v[1];
let remote_ref = r.peel_to_commit()?.id();
let ref_name = r.name().unwrap_or_default();

if !ref_name.starts_with("refs/remotes/") {
debug!("skipping non-remote ref {}", &ref_name);
// filter out the specific remote note ref for "note_suffix"
if !ref_name.ends_with(&ref_suffix) || !ref_name.starts_with("refs/remotes/") {
debug!("skipping ref {}", &ref_name);
continue;
}

info!("XET sync_note_refs_to_local: updating {}", &ref_name);

if !remote_ref.is_empty() {
self.run_git_checked_in_repo(
"notes",
&[&format!("--ref={notes_ref_suffix}"), "merge", remote_ref],
)?;
}
self.run_git_checked_in_repo(
"notes",
&[
&format!("--ref={notes_ref_suffix}"),
"merge",
&remote_ref.to_string(),
],
)?;
}
}

Expand Down Expand Up @@ -2243,4 +2250,26 @@ mod git_repo_tests {
assert!(GITATTRIBUTES_TEST_REGEX.is_match("*.gitattributes filter="));
assert!(GITATTRIBUTES_TEST_REGEX.is_match("*.xet/** filter="));
}

#[test]
fn test_show_ref() -> anyhow::Result<()> {
let path = Path::new("/Users/di/tt/bsf13");
let repo = git2::Repository::discover(path)?;
let references = repo.references()?;

for reference in references {
if let Ok(r) = reference {
let name = r.name();

if let Some(name) = name {
if name.ends_with("notes/xet_alt/merkledb") {
let commit_oid = r.peel_to_commit()?.id();
eprintln!("{commit_oid:?} {name:?}");
}
}
}
}

Ok(())
}
}