zoekt-indexserver: Per-Entry Branch Config

cmd/zoekt-indexserver/config.go

@@ -59,6 +59,11 @@ type ConfigEntry struct {
 	ExcludeUserRepos       bool
 	Forks                  bool
 	Visibility             []string
+
+	// Branches is a comma-separated list of branches to index, with support
+	// for wildcards (e.g. "main,release-*"). Defaults to HEAD if unset.
+	Branches     string
+	BranchPrefix string
 }
 
 func randomize(entries []ConfigEntry) []ConfigEntry {
@@ -338,6 +343,13 @@ func executeMirror(cfg []ConfigEntry, repoDir string, pendingRepos chan<- string
 			continue
 		}
 
+		if c.Branches != "" {
+			cmd.Args = append(cmd.Args, "-branches", c.Branches)
+		}
+		if c.BranchPrefix != "" {
+			cmd.Args = append(cmd.Args, "-branch-prefix", c.BranchPrefix)
+		}
+
 		stdout, _ := loggedRun(cmd)
 
 		for _, fn := range bytes.Split(stdout, []byte{'\n'}) {

cmd/zoekt-indexserver/main.go

@@ -163,6 +163,21 @@ func indexPendingRepo(dir, indexDir, repoDir string, opts *Options) {
 		"-index", indexDir,
 		"-incremental",
 	}
+
+	// Read zoekt.branches-to-index from the repo's git config. If set, pass
+	// it to zoekt-git-index as -branches along with -allow_missing_branches.
+	// The value is a comma-separated list of branch names (default: HEAD).
+	if branchOut, err := exec.Command("git", "--git-dir", dir, "config", "zoekt.branches-to-index").Output(); err == nil {
+		if branches := strings.TrimSpace(string(branchOut)); branches != "" {
+			args = append(args, "-branches", branches, "-allow_missing_branches")
+		}
+	}
+	if prefixOut, err := exec.Command("git", "--git-dir", dir, "config", "zoekt.branch-prefix").Output(); err == nil {
+		if prefix := strings.TrimSpace(string(prefixOut)); prefix != "" {
+			args = append(args, "-prefix", prefix)
+		}
+	}
+
 	args = append(args, opts.indexFlags...)
 	args = append(args, dir)
 	cmd := exec.CommandContext(ctx, "zoekt-git-index", args...)

cmd/zoekt-mirror-bitbucket-server/main.go

@@ -43,6 +43,8 @@ func main() {
 	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
 	projectType := flag.String("type", "", "only clone repos whose type matches the given string. "+
 		"Type can be either NORMAl or PERSONAL. Clones projects of both types if not set.")
+	branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.")
+	branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".")
 	flag.Parse()
 
 	if *serverUrl == "" {
@@ -134,7 +136,7 @@ func main() {
 	}
 	repos = trimmed
 
-	if err := cloneRepos(destDir, rootURL.Host, repos, password); err != nil {
+	if err := cloneRepos(destDir, rootURL.Host, repos, password, *branches, *branchPrefix); err != nil {
 		log.Fatalf("cloneRepos: %v", err)
 	}
 
@@ -235,14 +237,20 @@ func getProjectRepos(client bitbucketv1.APIClient, projectName string) ([]bitbuc
 	return allRepos, nil
 }
 
-func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string) error {
+func cloneRepos(destDir string, host string, repos []bitbucketv1.Repository, password string, branches string, branchPrefix string) error {
 	for _, r := range repos {
 		fullName := filepath.Join(r.Project.Key, r.Slug)
 		config := map[string]string{
 			"zoekt.web-url-type": "bitbucket-server",
 			"zoekt.web-url":      r.Links.Self[0].Href,
 			"zoekt.name":         filepath.Join(host, fullName),
 		}
+		if branches != "" {
+			config["zoekt.branches-to-index"] = branches
+		}
+		if branchPrefix != "" {
+			config["zoekt.branch-prefix"] = branchPrefix
+		}
 
 		httpsCloneUrl := ""
 		for _, cloneUrl := range r.Links.Clone {

cmd/zoekt-mirror-gerrit/main.go

@@ -98,6 +98,8 @@ func main() {
 	fetchMetaConfig := flag.Bool("fetch-meta-config", false, "fetch gerrit meta/config branch")
 	httpCrendentialsPath := flag.String("http-credentials", "", "path to a file containing http credentials stored like 'user:password'.")
 	active := flag.Bool("active", false, "mirror only active projects")
+	branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.")
+	branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".")
 	flag.Parse()
 
 	if len(flag.Args()) < 1 {
@@ -201,6 +203,13 @@ func main() {
 			"zoekt.public":         marshalBool(v.State != "HIDDEN"),
 		}
 
+		if *branches != "" {
+			config["zoekt.branches-to-index"] = *branches
+		}
+		if *branchPrefix != "" {
+			config["zoekt.branch-prefix"] = *branchPrefix
+		}
+
 		for _, wl := range v.WebLinks {
 			// default gerrit gitiles config is named browse, and does not include
 			// root domain name in it. Cheating.

cmd/zoekt-mirror-gitea/main.go

@@ -65,6 +65,8 @@ func main() {
 	excludeTopics := topicsFlag{}
 	flag.Var(&excludeTopics, "exclude_topic", "don't clone repos whose have one of given topics. You can add multiple topics by setting this more than once.")
 	noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
+	branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.")
+	branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".")
 
 	flag.Parse()
 
@@ -147,7 +149,7 @@ func main() {
 		repos = trimmed
 	}
 
-	if err := cloneRepos(destDir, repos); err != nil {
+	if err := cloneRepos(destDir, repos, *branches, *branchPrefix); err != nil {
 		log.Fatalf("cloneRepos: %v", err)
 	}
 
@@ -252,7 +254,7 @@ func getUserRepos(client *gitea.Client, user string, reposFilters reposFilters)
 	return allRepos, nil
 }
 
-func cloneRepos(destDir string, repos []*gitea.Repository) error {
+func cloneRepos(destDir string, repos []*gitea.Repository, branches string, branchPrefix string) error {
 	for _, r := range repos {
 		host, err := url.Parse(r.HTMLURL)
 		if err != nil {
@@ -274,6 +276,12 @@ func cloneRepos(destDir string, repos []*gitea.Repository) error {
 			"zoekt.fork":     marshalBool(r.Fork),
 			"zoekt.public":   marshalBool(r.Private || r.Internal), // count internal repos as private
 		}
+		if branches != "" {
+			config["zoekt.branches-to-index"] = branches
+		}
+		if branchPrefix != "" {
+			config["zoekt.branch-prefix"] = branchPrefix
+		}
 		dest, err := gitindex.CloneRepo(destDir, r.FullName, r.CloneURL, config)
 		if err != nil {
 			return err

cmd/zoekt-mirror-github/main.go

@@ -71,6 +71,8 @@ func main() {
 	noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
 	visibility := topicsFlag{}
 	flag.Var(&visibility, "visibility", "filter repos by visibility (public, private, internal). You can add multiple values by setting this more than once.")
+	branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.")
+	branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".")
 
 	flag.Parse()
 
@@ -154,7 +156,7 @@ func main() {
 		repos = trimmed
 	}
 
-	if err := cloneRepos(destDir, repos); err != nil {
+	if err := cloneRepos(destDir, repos, *branches, *branchPrefix); err != nil {
 		log.Fatalf("cloneRepos: %v", err)
 	}
 
@@ -299,7 +301,7 @@ func itoa(p *int) string {
 	return ""
 }
 
-func cloneRepos(destDir string, repos []*github.Repository) error {
+func cloneRepos(destDir string, repos []*github.Repository, branches string, branchPrefix string) error {
 	for _, r := range repos {
 		host, err := url.Parse(*r.HTMLURL)
 		if err != nil {
@@ -320,6 +322,12 @@ func cloneRepos(destDir string, repos []*github.Repository) error {
 			"zoekt.fork":     marshalBool(r.Fork != nil && *r.Fork),
 			"zoekt.public":   marshalBool(r.Private == nil || !*r.Private),
 		}
+		if branches != "" {
+			config["zoekt.branches-to-index"] = branches
+		}
+		if branchPrefix != "" {
+			config["zoekt.branch-prefix"] = branchPrefix
+		}
 		dest, err := gitindex.CloneRepo(destDir, *r.FullName, *r.CloneURL, config)
 		if err != nil {
 			return err

cmd/zoekt-mirror-gitiles/main.go

@@ -40,6 +40,8 @@ func main() {
 	namePattern := flag.String("name", "", "only clone repos whose name matches the regexp.")
 	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
 	hostType := flag.String("type", "gitiles", "which webserver to crawl. Choices: gitiles, cgit")
+	branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.")
+	branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".")
 	flag.Parse()
 
 	if len(flag.Args()) < 1 {
@@ -90,6 +92,12 @@ func main() {
 			"zoekt.web-url-type": target.webURLType,
 			"zoekt.name":         fullName,
 		}
+		if *branches != "" {
+			config["zoekt.branches-to-index"] = *branches
+		}
+		if *branchPrefix != "" {
+			config["zoekt.branch-prefix"] = *branchPrefix
+		}
 
 		dest, err := gitindex.CloneRepo(*dest, fullName, target.cloneURL, config)
 		if err != nil {

cmd/zoekt-mirror-gitlab/main.go

@@ -53,6 +53,8 @@ func main() {
 	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
 	lastActivityAfter := flag.String("last_activity_after", "", "only mirror repos that have been active since this date (format: 2006-01-02).")
 	noArchived := flag.Bool("no_archived", false, "mirror only projects that are not archived")
+	branches := flag.String("branches", "", "comma-separated list of branches to index (e.g. \"main,release-*\"). Defaults to HEAD if unset.")
+	branchPrefix := flag.String("branch-prefix", "", "prefix for branch names (e.g. \"refs/tags/\" to index tags). Defaults to \"refs/heads/\".")
 
 	flag.Parse()
 
@@ -149,7 +151,7 @@ func main() {
 		}
 		gitlabProjects = trimmed
 	}
-	fetchProjects(destDir, apiToken, gitlabProjects)
+	fetchProjects(destDir, apiToken, gitlabProjects, *branches, *branchPrefix)
 
 	if *deleteRepos {
 		if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil {
@@ -181,7 +183,7 @@ func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gi
 	return nil
 }
 
-func fetchProjects(destDir, token string, projects []*gitlab.Project) {
+func fetchProjects(destDir, token string, projects []*gitlab.Project, branches string, branchPrefix string) {
 	for _, p := range projects {
 		u, err := url.Parse(p.HTTPURLToRepo)
 		if err != nil {
@@ -200,6 +202,12 @@ func fetchProjects(destDir, token string, projects []*gitlab.Project) {
 			"zoekt.fork":     marshalBool(p.ForkedFromProject != nil),
 			"zoekt.public":   marshalBool(p.Visibility == gitlab.PublicVisibility),
 		}
+		if branches != "" {
+			config["zoekt.branches-to-index"] = branches
+		}
+		if branchPrefix != "" {
+			config["zoekt.branch-prefix"] = branchPrefix
+		}
 
 		cloneURL := p.HTTPURLToRepo
 		dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config)