From b66a7292c6a0965efc75397c78dc0b90ba27f243 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:10:48 +0000 Subject: [PATCH] builder-agent: use /var/lib/buildkit directly when already mounted If the BuildKit root is already a mountpoint (e.g. a dedicated volume), use it as-is instead of shadowing it. Otherwise keep the existing 3 GB tmpfs mount required for the native overlayfs snapshotter on the overlayfs rootfs. --- lib/builds/builder_agent/main.go | 61 +++++++++++++++++++++------ lib/builds/builder_agent/main_test.go | 57 +++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 lib/builds/builder_agent/main_test.go diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index 2540d7e01..e8559084b 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -856,21 +856,10 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st buildkitdConfig := "/home/builder/.config/buildkit/buildkitd.toml" log.Printf("Using buildkitd config: %s", buildkitdConfig) - // Mount a tmpfs for BuildKit's data directory. - // The VM rootfs is an overlayfs (read-only ext4 + writable ext4 upper layer). - // BuildKit's native overlayfs snapshotter creates char device 0:0 for whiteout - // markers, but mknod(char 0:0) fails on an overlayfs mount because the kernel - // treats it as an overlayfs whiteout rather than a regular device node. - // Using tmpfs avoids this nested-overlayfs conflict. buildkitRoot := "/var/lib/buildkit" - if err := os.MkdirAll(buildkitRoot, 0755); err != nil { - return "", "", fmt.Errorf("create buildkit root dir: %w", err) + if err := ensureBuildkitRoot(buildkitRoot, isMountPoint, mountBuildkitTmpfs); err != nil { + return "", "", err } - mountCmd := exec.Command("mount", "-t", "tmpfs", "-o", "size=3G", "tmpfs", buildkitRoot) - if output, err := mountCmd.CombinedOutput(); err != nil { - return "", "", fmt.Errorf("mount tmpfs at %s (required for native overlayfs snapshotter): %v: %s", buildkitRoot, err, output) - } - log.Printf("Mounted tmpfs at %s for BuildKit snapshotter", buildkitRoot) log.Printf("Running: buildctl-daemonless.sh %s", strings.Join(args, " ")) @@ -911,6 +900,52 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st return digest, buildLogs.String(), nil } +// ensureBuildkitRoot prepares BuildKit's data directory. If root is already a +// mountpoint it is used directly. Otherwise a tmpfs is mounted there: the VM +// rootfs is an overlayfs (read-only ext4 + writable ext4 upper layer) and +// BuildKit's native overlayfs snapshotter creates char device 0:0 for whiteout +// markers, but mknod(char 0:0) fails on an overlayfs mount because the kernel +// treats it as an overlayfs whiteout rather than a regular device node. tmpfs +// avoids this nested-overlayfs conflict. +func ensureBuildkitRoot(root string, isMounted func(string) (bool, error), mountTmpfs func(string) error) error { + mounted, err := isMounted(root) + if err != nil { + return fmt.Errorf("check mountpoint %s: %w", root, err) + } + if mounted { + log.Printf("Using existing mount at %s for BuildKit", root) + return nil + } + if err := os.MkdirAll(root, 0755); err != nil { + return fmt.Errorf("create buildkit root dir: %w", err) + } + return mountTmpfs(root) +} + +func mountBuildkitTmpfs(root string) error { + mountCmd := exec.Command("mount", "-t", "tmpfs", "-o", "size=3G", "tmpfs", root) + if output, err := mountCmd.CombinedOutput(); err != nil { + return fmt.Errorf("mount tmpfs at %s (required for native overlayfs snapshotter): %v: %s", root, err, output) + } + log.Printf("Mounted tmpfs at %s for BuildKit snapshotter", root) + return nil +} + +// isMountPoint reports whether path is a mountpoint according to /proc/self/mounts. +func isMountPoint(path string) (bool, error) { + data, err := os.ReadFile("/proc/self/mounts") + if err != nil { + return false, err + } + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[1] == path { + return true, nil + } + } + return false, nil +} + func extractDigest(metadataPath string) (string, error) { data, err := os.ReadFile(metadataPath) if err != nil { diff --git a/lib/builds/builder_agent/main_test.go b/lib/builds/builder_agent/main_test.go new file mode 100644 index 000000000..986750396 --- /dev/null +++ b/lib/builds/builder_agent/main_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnsureBuildkitRootAlreadyMounted(t *testing.T) { + root := filepath.Join(t.TempDir(), "buildkit") + + mountCalled := false + err := ensureBuildkitRoot(root, + func(path string) (bool, error) { + assert.Equal(t, root, path) + return true, nil + }, + func(path string) error { + mountCalled = true + return nil + }, + ) + + require.NoError(t, err) + assert.False(t, mountCalled, "tmpfs mount must not be attempted when root is already a mountpoint") +} + +func TestEnsureBuildkitRootMountsTmpfsWhenUnmounted(t *testing.T) { + root := filepath.Join(t.TempDir(), "buildkit") + + var mountedPath string + err := ensureBuildkitRoot(root, + func(path string) (bool, error) { return false, nil }, + func(path string) error { + mountedPath = path + return nil + }, + ) + + require.NoError(t, err) + assert.Equal(t, root, mountedPath) + assert.DirExists(t, root) +} + +func TestEnsureBuildkitRootPropagatesMountFailure(t *testing.T) { + root := filepath.Join(t.TempDir(), "buildkit") + + err := ensureBuildkitRoot(root, + func(path string) (bool, error) { return false, nil }, + func(path string) error { return errors.New("mount failed") }, + ) + + require.ErrorContains(t, err, "mount failed") +}