diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go index ff484c50d..6c0cc1c03 100644 --- a/cmd/zoekt-indexserver/config.go +++ b/cmd/zoekt-indexserver/config.go @@ -59,6 +59,11 @@ type ConfigEntry struct { ExcludeUserRepos bool Forks bool Visibility []string + + // Branches is a comma-separated list of branches to index, with support + // for wildcards (e.g. "main,release-*"). Defaults to HEAD if unset. + Branches string + BranchPrefix string } func randomize(entries []ConfigEntry) []ConfigEntry { @@ -338,6 +343,13 @@ func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string continue } + if c.Branches != "" { + cmd.Args = append(cmd.Args, "-branches", c.Branches) + } + if c.BranchPrefix != "" { + cmd.Args = append(cmd.Args, "-branch-prefix", c.BranchPrefix) + } + stdout, _ := loggedRun(cmd) for _, fn := range bytes.Split(stdout, []byte{'\n'}) { diff --git a/cmd/zoekt-indexserver/main.go b/cmd/zoekt-indexserver/main.go index 6e54ca710..315d5592b 100644 --- a/cmd/zoekt-indexserver/main.go +++ b/cmd/zoekt-indexserver/main.go @@ -163,6 +163,21 @@ func indexPendingRepo(dir, indexDir, repoDir string, opts *Options) { "-index", indexDir, "-incremental", } + + // Read zoekt.branches-to-index from the repo's git config. If set, pass + // it to zoekt-git-index as -branches along with -allow_missing_branches. + // The value is a comma-separated list of branch names (default: HEAD). + if branchOut, err := exec.Command("git", "--git-dir", dir, "config", "zoekt.branches-to-index").Output(); err == nil { + if branches := strings.TrimSpace(string(branchOut)); branches != "" { + args = append(args, "-branches", branches, "-allow_missing_branches") + } + } + if prefixOut, err := exec.Command("git", "--git-dir", dir, "config", "zoekt.branch-prefix").Output(); err == nil { + if prefix := strings.TrimSpace(string(prefixOut)); prefix != "" { + args = append(args, "-prefix", prefix) + } + } + args = append(args, opts.indexFlags...) args = append(args, dir) cmd := exec.CommandContext(ctx, "zoekt-git-index", args...) diff --git a/cmd/zoekt-mirror-bitbucket-server/main.go b/cmd/zoekt-mirror-bitbucket-server/main.go index f5f25c198..520a2e12a 100644 --- a/cmd/zoekt-mirror-bitbucket-server/main.go +++ b/cmd/zoekt-mirror-bitbucket-server/main.go @@ -43,6 +43,8 @@ func main() { excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+ "Type can be either NORMAl or PERSONAL. Clones projects of both types if not set.") + branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.") + branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".") flag.Parse() if *serverUrl == "" { @@ -134,7 +136,7 @@ func main() { } repos = trimmed - if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil { + if err := cloneRepos(destDir, rootURL.Host, repos, password, *branches, *branchPrefix); err != nil { log.Fatalf("cloneRepos: %v", err) } @@ -235,7 +237,7 @@ func getProjectRepos(client bitbucketv1.APIClient, projectName string) ([]bitbuc return allRepos, nil } -func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string) error { +func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string, branches string, branchPrefix string) error { for _, r := range repos { fullName := filepath.Join(r.Project.Key, r.Slug) config := map[string]string{ @@ -243,6 +245,12 @@ func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, pas "zoekt.web-url": r.Links.Self[0].Href, "zoekt.name": filepath.Join(host, fullName), } + if branches != "" { + config["zoekt.branches-to-index"] = branches + } + if branchPrefix != "" { + config["zoekt.branch-prefix"] = branchPrefix + } httpsCloneUrl := "" for _, cloneUrl := range r.Links.Clone { diff --git a/cmd/zoekt-mirror-gerrit/main.go b/cmd/zoekt-mirror-gerrit/main.go index 71d45c812..c9715a52f 100644 --- a/cmd/zoekt-mirror-gerrit/main.go +++ b/cmd/zoekt-mirror-gerrit/main.go @@ -98,6 +98,8 @@ func main() { fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch") httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.") active := flag.Bool("active", false, "mirror only active projects") + branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.") + branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".") flag.Parse() if len(flag.Args()) < 1 { @@ -201,6 +203,13 @@ func main() { "zoekt.public": marshalBool(v.State != "HIDDEN"), } + if *branches != "" { + config["zoekt.branches-to-index"] = *branches + } + if *branchPrefix != "" { + config["zoekt.branch-prefix"] = *branchPrefix + } + for _, wl := range v.WebLinks { // default gerrit gitiles config is named browse, and does not include // root domain name in it. Cheating. diff --git a/cmd/zoekt-mirror-gitea/main.go b/cmd/zoekt-mirror-gitea/main.go index 71e8814b5..49abec9a1 100644 --- a/cmd/zoekt-mirror-gitea/main.go +++ b/cmd/zoekt-mirror-gitea/main.go @@ -65,6 +65,8 @@ func main() { excludeTopics := topicsFlag{} flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.") noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived") + branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.") + branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".") flag.Parse() @@ -147,7 +149,7 @@ func main() { repos = trimmed } - if err := cloneRepos(destDir, repos); err != nil { + if err := cloneRepos(destDir, repos, *branches, *branchPrefix); err != nil { log.Fatalf("cloneRepos: %v", err) } @@ -252,7 +254,7 @@ func getUserRepos(client *gitea.Client, user string, reposFilters reposFilters) return allRepos, nil } -func cloneRepos(destDir string, repos []*gitea.Repository) error { +func cloneRepos(destDir string, repos []*gitea.Repository, branches string, branchPrefix string) error { for _, r := range repos { host, err := url.Parse(r.HTMLURL) if err != nil { @@ -274,6 +276,12 @@ func cloneRepos(destDir string, repos []*gitea.Repository) error { "zoekt.fork": marshalBool(r.Fork), "zoekt.public": marshalBool(r.Private || r.Internal), // count internal repos as private } + if branches != "" { + config["zoekt.branches-to-index"] = branches + } + if branchPrefix != "" { + config["zoekt.branch-prefix"] = branchPrefix + } dest, err := gitindex.CloneRepo(destDir, r.FullName, r.CloneURL, config) if err != nil { return err diff --git a/cmd/zoekt-mirror-github/main.go b/cmd/zoekt-mirror-github/main.go index 4433fa017..4f0c8e7d0 100644 --- a/cmd/zoekt-mirror-github/main.go +++ b/cmd/zoekt-mirror-github/main.go @@ -71,6 +71,8 @@ func main() { noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived") visibility := topicsFlag{} flag.Var(&visibility, "visibility", "filter repos by visibility (public, private, internal). You can add multiple values by setting this more than once.") + branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.") + branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".") flag.Parse() @@ -154,7 +156,7 @@ func main() { repos = trimmed } - if err := cloneRepos(destDir, repos); err != nil { + if err := cloneRepos(destDir, repos, *branches, *branchPrefix); err != nil { log.Fatalf("cloneRepos: %v", err) } @@ -299,7 +301,7 @@ func itoa(p *int) string { return "" } -func cloneRepos(destDir string, repos []*github.Repository) error { +func cloneRepos(destDir string, repos []*github.Repository, branches string, branchPrefix string) error { for _, r := range repos { host, err := url.Parse(*r.HTMLURL) if err != nil { @@ -320,6 +322,12 @@ func cloneRepos(destDir string, repos []*github.Repository) error { "zoekt.fork": marshalBool(r.Fork != nil && *r.Fork), "zoekt.public": marshalBool(r.Private == nil || !*r.Private), } + if branches != "" { + config["zoekt.branches-to-index"] = branches + } + if branchPrefix != "" { + config["zoekt.branch-prefix"] = branchPrefix + } dest, err := gitindex.CloneRepo(destDir, *r.FullName, *r.CloneURL, config) if err != nil { return err diff --git a/cmd/zoekt-mirror-gitiles/main.go b/cmd/zoekt-mirror-gitiles/main.go index 4625b22fe..d86afc1ef 100644 --- a/cmd/zoekt-mirror-gitiles/main.go +++ b/cmd/zoekt-mirror-gitiles/main.go @@ -40,6 +40,8 @@ func main() { namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.") excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") hostType := flag.String("type", "gitiles", "which webserver to crawl. Choices: gitiles, cgit") + branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.") + branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".") flag.Parse() if len(flag.Args()) < 1 { @@ -90,6 +92,12 @@ func main() { "zoekt.web-url-type": target.webURLType, "zoekt.name": fullName, } + if *branches != "" { + config["zoekt.branches-to-index"] = *branches + } + if *branchPrefix != "" { + config["zoekt.branch-prefix"] = *branchPrefix + } dest, err := gitindex.CloneRepo(*dest, fullName, target.cloneURL, config) if err != nil { diff --git a/cmd/zoekt-mirror-gitlab/main.go b/cmd/zoekt-mirror-gitlab/main.go index 5c0fa46dd..41569c9a4 100644 --- a/cmd/zoekt-mirror-gitlab/main.go +++ b/cmd/zoekt-mirror-gitlab/main.go @@ -53,6 +53,8 @@ func main() { excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") lastActivityAfter := flag.String("last_activity_after", "", "only mirror repos that have been active since this date (format: 2006-01-02).") noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived") + branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.") + branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".") flag.Parse() @@ -149,7 +151,7 @@ func main() { } gitlabProjects = trimmed } - fetchProjects(destDir, apiToken, gitlabProjects) + fetchProjects(destDir, apiToken, gitlabProjects, *branches, *branchPrefix) if *deleteRepos { if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil { @@ -181,7 +183,7 @@ func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gi return nil } -func fetchProjects(destDir, token string, projects []*gitlab.Project) { +func fetchProjects(destDir, token string, projects []*gitlab.Project, branches string, branchPrefix string) { for _, p := range projects { u, err := url.Parse(p.HTTPURLToRepo) if err != nil { @@ -200,6 +202,12 @@ func fetchProjects(destDir, token string, projects []*gitlab.Project) { "zoekt.fork": marshalBool(p.ForkedFromProject != nil), "zoekt.public": marshalBool(p.Visibility == gitlab.PublicVisibility), } + if branches != "" { + config["zoekt.branches-to-index"] = branches + } + if branchPrefix != "" { + config["zoekt.branch-prefix"] = branchPrefix + } cloneURL := p.HTTPURLToRepo dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config)