diff --git a/cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go b/cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go index 7bf5ad5f8..06b5e10bd 100644 --- a/cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go +++ b/cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go @@ -26,6 +26,7 @@ import ( "github.com/moby/sys/reexec" "github.com/urfave/cli/v3" + "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" "github.com/NVIDIA/nvidia-container-toolkit/internal/ldconfig" "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" @@ -119,6 +120,7 @@ func (m command) run(_ *cli.Command, cfg *options) error { reexecUpdateLdCacheCommandName, cfg.ldconfigPath, containerRootDir, + m.compat32Requested(s), cfg.folders..., ) if err != nil { @@ -127,6 +129,26 @@ func (m command) run(_ *cli.Command, cfg *options) error { return runner.Run() } +// compat32Requested checks whether the container requested the compat32 +// driver capability. This mirrors the nvidia-container-cli, where the 32-bit +// libraries are associated with this capability. +// Note that the environment of the container is read from its OCI spec, which +// is not always accessible, and a container whose capabilities cannot be +// determined is treated as not having requested this one. +func (m command) compat32Requested(s *oci.State) bool { + env, err := s.GetEnv() + if err != nil { + m.logger.Debugf("Failed to get the container environment: %v", err) + return false + } + containerImage, err := image.New(image.WithEnv(env)) + if err != nil { + m.logger.Debugf("Failed to construct image from the container environment: %v", err) + return false + } + return containerImage.GetDriverCapabilities().Has(image.DriverCapabilityCompat32) +} + // updateLdCacheHandler wraps updateLdCache with error handling. func updateLdCacheHandler() { if err := updateLdCache(os.Args); err != nil { diff --git a/cmd/nvidia-ctk/cdi/generate/generate.go b/cmd/nvidia-ctk/cdi/generate/generate.go index 7c29e9dd9..d8ddfb89a 100644 --- a/cmd/nvidia-ctk/cdi/generate/generate.go +++ b/cmd/nvidia-ctk/cdi/generate/generate.go @@ -67,7 +67,8 @@ type options struct { disabledHooks []string enabledHooks []string - featureFlags []string + disableCompat32 bool + featureFlags []string csv struct { files []string @@ -236,6 +237,16 @@ func (m command) build() *cli.Command { Destination: &opts.enabledHooks, Sources: cli.EnvVars("NVIDIA_CTK_CDI_GENERATE_ENABLED_HOOKS"), }, + &cli.BoolFlag{ + Name: "disable-compat32", + Usage: "Exclude the 32-bit compatibility driver libraries installed on the host from the " + + "generated CDI specification. These are included by default and are only exposed to a " + + "container that can run 32-bit applications: one that requests the 'compat32' driver " + + "capability or ships a 32-bit dynamic linker, and never a container that uses musl. " + + "This is equivalent to specifying the '" + string(nvcdi.FeatureDisableCompat32Libraries) + "' feature flag.", + Destination: &opts.disableCompat32, + Sources: cli.EnvVars("NVIDIA_CTK_CDI_GENERATE_DISABLE_COMPAT32"), + }, &cli.StringSliceFlag{ Name: "feature-flag", Aliases: []string{"feature-flags"}, @@ -399,6 +410,10 @@ func (m command) generateSpecs(opts *options) ([]generatedSpecs, error) { nvcdi.WithNvmlLib(opts.nvmllib), } + if opts.disableCompat32 { + cdiOptions = append(cdiOptions, nvcdi.WithFeatureFlags(nvcdi.FeatureDisableCompat32Libraries)) + } + cdilib, err := nvcdi.New(cdiOptions...) if err != nil { return nil, fmt.Errorf("failed to create CDI library: %v", err) diff --git a/deployments/systemd/nvidia-cdi-refresh.env b/deployments/systemd/nvidia-cdi-refresh.env index 45315914b..4d3db686f 100644 --- a/deployments/systemd/nvidia-cdi-refresh.env +++ b/deployments/systemd/nvidia-cdi-refresh.env @@ -19,6 +19,11 @@ # uncomment the following line: # NVIDIA_CTK_CDI_OUTPUT_FILE_PATH=/var/run/cdi/nvidia.yaml +# The 32-bit compatibility driver libraries installed on the host are included +# in the generated CDI specification. To leave these out, uncomment the +# following line: +# NVIDIA_CTK_CDI_GENERATE_DISABLE_COMPAT32=true + # The service also runs # nvidia-ctk system create-device-nodes --control-devices --load-kernel-modules # before generating the CDI specification. Its driver and device roots can be diff --git a/internal/ldconfig/compat32.go b/internal/ldconfig/compat32.go new file mode 100644 index 000000000..ce7a57389 --- /dev/null +++ b/internal/ldconfig/compat32.go @@ -0,0 +1,93 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package ldconfig + +import ( + "debug/elf" + "path/filepath" + "runtime" +) + +// compat32Loaders maps the platform to the paths of the dynamic linkers that a +// container uses to run 32-bit applications. +var compat32Loaders = map[string][]string{ + "amd64": {"/lib/ld-linux.so.2", "/lib32/ld-linux.so.2"}, + "arm64": {"/lib/ld-linux-armhf.so.3", "/lib/ld-linux.so.3"}, +} + +// allowsCompat32 checks whether the 32-bit driver libraries are of use in the +// specified root. +// +// A container that uses musl never sees these. A musl .path file carries no +// architecture information and musl has no notion of a multiarch layout: it +// loads the first file matching the requested name and fails instead of +// searching on if that file is of another ELF class. +// +// Other containers see the 32-bit libraries if they requested the compat32 +// driver capability -- as is the case for the nvidia-container-cli -- or if +// they ship a dynamic linker for 32-bit applications. +func allowsCompat32(root string, requested bool) bool { + if arch, ok := muslArchs[runtime.GOARCH]; ok && isMusl(root, arch) { + return false + } + if requested { + return true + } + for _, loader := range compat32Loaders[runtime.GOARCH] { + if isFile(filepath.Join(root, loader)) { + return true + } + } + return false +} + +// excludeCompat32Directories returns the specified directories with those that +// hold 32-bit libraries removed. The directories are resolved relative to the +// specified root. +func excludeCompat32Directories(root string, dirs []string) []string { + var filtered []string + for _, dir := range dirs { + if isCompat32Dir(filepath.Join(root, dir)) { + continue + } + filtered = append(filtered, dir) + } + return filtered +} + +// isCompat32Dir checks whether the specified directory holds 32-bit libraries +// and no 64-bit ones. Files that are not ELF files are ignored, as are +// directories that hold no libraries at all. +// Note that both supported platforms are 64-bit. +func isCompat32Dir(dir string) bool { + var compat32 bool + libraries, _ := filepath.Glob(filepath.Join(dir, "lib?*.so*")) + for _, library := range libraries { + f, err := elf.Open(library) + if err != nil { + continue + } + class := f.Class + _ = f.Close() + if class == elf.ELFCLASS64 { + return false + } + compat32 = compat32 || class == elf.ELFCLASS32 + } + return compat32 +} diff --git a/internal/ldconfig/compat32_test.go b/internal/ldconfig/compat32_test.go new file mode 100644 index 000000000..397f48f56 --- /dev/null +++ b/internal/ldconfig/compat32_test.go @@ -0,0 +1,191 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package ldconfig + +import ( + "bytes" + "debug/elf" + "encoding/binary" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAllowsCompat32(t *testing.T) { + loaders, ok := compat32Loaders[runtime.GOARCH] + if !ok { + t.Skip("32-bit libraries are not handled on this platform") + } + muslArch := muslArchs[runtime.GOARCH] + + testCases := []struct { + description string + // compat32Loader adds a dynamic linker for 32-bit applications. + compat32Loader string + // isMusl adds the musl dynamic linker. + isMusl bool + // requested indicates that the compat32 driver capability was requested. + requested bool + expected bool + }{ + { + description: "64-bit-only container is skipped", + }, + { + description: "container with a 32-bit loader is allowed", + compat32Loader: loaders[0], + expected: true, + }, + { + description: "container that requested the capability is allowed", + requested: true, + expected: true, + }, + { + description: "musl container is skipped", + isMusl: true, + }, + { + description: "musl container that requested the capability is skipped", + isMusl: true, + requested: true, + }, + { + description: "musl container with a 32-bit loader is skipped", + isMusl: true, + compat32Loader: loaders[0], + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + root := t.TempDir() + if tc.compat32Loader != "" { + makeFile(t, filepath.Join(root, tc.compat32Loader)) + } + if tc.isMusl { + makeMuslLoader(t, root, muslArch) + } + + require.Equal(t, tc.expected, allowsCompat32(root, tc.requested)) + }) + } +} + +func TestAllowsCompat32DetectsAllLoaders(t *testing.T) { + for _, loader := range compat32Loaders[runtime.GOARCH] { + t.Run(loader, func(t *testing.T) { + root := t.TempDir() + makeFile(t, filepath.Join(root, loader)) + + require.True(t, allowsCompat32(root, false)) + }) + } +} + +func TestExcludeCompat32Directories(t *testing.T) { + root := t.TempDir() + makeLibDir(t, root, "/native", elf.ELFCLASS64) + makeLibDir(t, root, "/compat32", elf.ELFCLASS32) + makeLibDir(t, root, "/mixed", elf.ELFCLASS64, elf.ELFCLASS32) + makeLibDir(t, root, "/empty") + notALib := makeLibDir(t, root, "/not-a-lib") + require.NoError(t, os.WriteFile(filepath.Join(notALib, "libnotelf.so.1"), []byte("#!/bin/sh\n"), 0o600)) + + testCases := []struct { + description string + dirs []string + expected []string + }{ + { + description: "32-bit dirs are removed", + dirs: []string{"/native", "/compat32"}, + expected: []string{"/native"}, + }, + { + description: "dirs with libraries of both classes are kept", + dirs: []string{"/mixed"}, + expected: []string{"/mixed"}, + }, + { + description: "dirs without libraries are kept", + dirs: []string{"/empty", "/not-a-lib", "/does-not-exist"}, + expected: []string{"/empty", "/not-a-lib", "/does-not-exist"}, + }, + { + description: "the order of the remaining dirs is maintained", + dirs: []string{"/compat32", "/mixed", "/native"}, + expected: []string{"/mixed", "/native"}, + }, + { + description: "32-bit dirs alone leave no dirs", + dirs: []string{"/compat32"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + require.Equal(t, tc.expected, excludeCompat32Directories(root, tc.dirs)) + }) + } +} + +// makeLibDir creates a directory in the specified root holding a library of +// each of the specified ELF classes. +func makeLibDir(t *testing.T, root string, dir string, classes ...elf.Class) string { + t.Helper() + + path := filepath.Join(root, dir) + require.NoError(t, os.MkdirAll(path, 0o755)) + for _, class := range classes { + name := "libnvidia-" + class.String() + ".so.999.88.77" + require.NoError(t, os.WriteFile(filepath.Join(path, name), elfFile(t, class), 0o600)) + } + return path +} + +// makeFile creates an empty file at the specified path. +func makeFile(t *testing.T, path string) { + t.Helper() + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, nil, 0o600)) +} + +// elfFile returns a minimal ELF file of the specified class: a header without +// program or section headers. +func elfFile(t *testing.T, class elf.Class) []byte { + t.Helper() + + var ident [elf.EI_NIDENT]byte + copy(ident[:], elf.ELFMAG) + ident[elf.EI_CLASS] = byte(class) + ident[elf.EI_DATA] = byte(elf.ELFDATA2LSB) + ident[elf.EI_VERSION] = byte(elf.EV_CURRENT) + + var header any = elf.Header64{Ident: ident, Version: uint32(elf.EV_CURRENT)} + if class == elf.ELFCLASS32 { + header = elf.Header32{Ident: ident, Version: uint32(elf.EV_CURRENT)} + } + var contents bytes.Buffer + require.NoError(t, binary.Write(&contents, binary.LittleEndian, header)) + return contents.Bytes() +} diff --git a/internal/ldconfig/ldconfig.go b/internal/ldconfig/ldconfig.go index c761efc42..3c3812385 100644 --- a/internal/ldconfig/ldconfig.go +++ b/internal/ldconfig/ldconfig.go @@ -63,11 +63,14 @@ type Ldconfig struct { isDebianLikeHost bool isDebianLikeContainer bool noPivotRoot bool + compat32Requested bool directories []string } // NewRunner creates an exec.Cmd that can be used to run ldconfig. -func NewRunner(id string, ldconfigPath string, containerRoot string, additionalargs ...string) (*exec.Cmd, error) { +// The compat32Requested argument indicates that the container requested the +// compat32 driver capability. +func NewRunner(id string, ldconfigPath string, containerRoot string, compat32Requested bool, additionalargs ...string) (*exec.Cmd, error) { args := []string{ id, "--ldconfig-path", strings.TrimPrefix(config.NormalizeLDConfigPath("@"+ldconfigPath), "@"), @@ -81,6 +84,10 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala args = append(args, "--no-pivot") } + if compat32Requested { + args = append(args, "--compat32") + } + args = append(args, additionalargs...) return createReexecCommand(args) @@ -104,6 +111,7 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala // as opposed to non-Debian-like (e.g. RHEL, Fedora) // See https://github.com/NVIDIA/nvidia-container-toolkit/pull/1444 // --no-pivot pivot_root should not be used to provide process isolation. +// --compat32 the container requested the compat32 driver capability. // // The remaining args are folders where soname symlinks need to be created. func NewFromArgs(args ...string) (*Ldconfig, error) { @@ -118,6 +126,7 @@ This allows us to handle the case where there are differences in behavior between the ldconfig from the host (as executed from an update-ldcache hook) and ldconfig in the container. Such differences include system search paths.`) noPivot := fs.Bool("no-pivot", false, "don't use pivot_root to perform isolation") + compat32 := fs.Bool("compat32", false, "the container requested the compat32 driver capability") if err := fs.Parse(args[1:]); err != nil { return nil, err } @@ -130,11 +139,12 @@ ldconfig in the container. Such differences include system search paths.`) } l := &Ldconfig{ - ldconfigPath: *ldconfigPath, - inRoot: *containerRoot, - isDebianLikeHost: *isDebianLikeHost, - noPivotRoot: *noPivot, - directories: fs.Args(), + ldconfigPath: *ldconfigPath, + inRoot: *containerRoot, + isDebianLikeHost: *isDebianLikeHost, + noPivotRoot: *noPivot, + compat32Requested: *compat32, + directories: fs.Args(), } return l, nil } @@ -154,7 +164,16 @@ func (l *Ldconfig) UpdateLDCache() error { return fmt.Errorf("failed to ensure ld.so.conf file: %w", err) } - filteredDirectories, err := l.filterDirectories(defaultTopLevelLdsoconfFilePath, l.directories...) + // The 32-bit libraries are only of use to a container that can run 32-bit + // applications and are left out of the search paths of the others. Since + // we have pivoted to the container root, these paths are resolved + // relative to "/". + directories := l.directories + if !allowsCompat32("/", l.compat32Requested) { + directories = excludeCompat32Directories("/", directories) + } + + filteredDirectories, err := l.filterDirectories(defaultTopLevelLdsoconfFilePath, directories...) if err != nil { return err } @@ -182,8 +201,10 @@ func (l *Ldconfig) UpdateLDCache() error { return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdSystemDirsFilenamePattern, err) } - // Also output the folders to the alpine .path file as required. - if err := createMuslPathFileIfRequired(append(filteredDirectories, systemSearchPaths...)...); err != nil { + // Also output the folders to the musl .path file as required. + // Note that musl does not process the ld.so.conf files and the directories + // that were filtered against these are therefore included here too. + if err := createMuslPathFileIfRequired("/", directories, systemSearchPaths); err != nil { return fmt.Errorf("failed to update .path file for musl: %w", err) } @@ -374,45 +395,6 @@ func processLdsoconfFile(ldsoconfFilename string) ([]string, []string, error) { return directories, includedFilenames, nil } -// createMuslPathFileIfRequired creates a musl .path file that allows libraries -// from the specified directories to be discovered on the system. -// This is required because systems that use musl do not rely on the ldcache to -// discover libraries. -func createMuslPathFileIfRequired(dirs ...string) error { - if len(dirs) == 0 || !isMusl() { - return nil - } - - var pathFileName string - switch runtime.GOARCH { - case "amd64": - pathFileName = "/etc/ld-musl-x86_64.path" - case "arm64": - pathFileName = "/etc/ld-musl-aarch64.path" - } - - pathFile, err := os.OpenFile(pathFileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644) - if err != nil { - return fmt.Errorf("could not open .path file: %w", err) - } - defer func() { - _ = pathFile.Close() - }() - - return outputListToFile(pathFile, dirs...) -} - -// isMusl checks whether the container is running musl instead of glibc. -// Note that for the time being we only check whether `/etc/alpine-release` is -// present in the container. -func isMusl() bool { - info, err := os.Stat("/etc/alpine-release") - if err != nil { - return false - } - return !info.IsDir() -} - // isDebianLike returns true if a Debian-like distribution is detected. // Debian-like distributions include Debian and Ubuntu, whereas non-Debian-like // distributions include RHEL and Fedora. diff --git a/internal/ldconfig/ldconfig_test.go b/internal/ldconfig/ldconfig_test.go index c983e7f4f..2b5f942ef 100644 --- a/internal/ldconfig/ldconfig_test.go +++ b/internal/ldconfig/ldconfig_test.go @@ -26,6 +26,43 @@ import ( "github.com/stretchr/testify/require" ) +func TestNewFromArgs(t *testing.T) { + requiredArgs := []string{ + "reexec-update-ldcache", + "--ldconfig-path", "/sbin/ldconfig", + "--container-root", "/container/root", + } + + testCases := []struct { + description string + args []string + expectedCompat32Requested bool + expectedDirectories []string + }{ + { + description: "folders are parsed", + args: []string{"/usr/lib/x86_64-linux-gnu", "/usr/lib/i386-linux-gnu"}, + expectedDirectories: []string{"/usr/lib/x86_64-linux-gnu", "/usr/lib/i386-linux-gnu"}, + }, + { + description: "compat32 is requested", + args: []string{"--compat32", "/usr/lib/i386-linux-gnu"}, + expectedCompat32Requested: true, + expectedDirectories: []string{"/usr/lib/i386-linux-gnu"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + l, err := NewFromArgs(append(requiredArgs, tc.args...)...) + require.NoError(t, err) + + require.Equal(t, tc.expectedCompat32Requested, l.compat32Requested) + require.Equal(t, tc.expectedDirectories, l.directories) + }) + } +} + func TestFilterDirectories(t *testing.T) { const topLevelConf = "TOPLEVEL.conf" diff --git a/internal/ldconfig/musl.go b/internal/ldconfig/musl.go new file mode 100644 index 000000000..ab16e3a62 --- /dev/null +++ b/internal/ldconfig/musl.go @@ -0,0 +1,110 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package ldconfig + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "strings" +) + +// muslArchs maps the platform to the musl architecture name, which names the +// dynamic linker, /lib/ld-musl-.so.1, and its .path file, +// /etc/ld-musl-.path. +var muslArchs = map[string]string{ + "amd64": "x86_64", + "arm64": "aarch64", +} + +// muslDefaultSearchPath is searched by musl when no .path file exists. +// Creating the file replaces it, so it is written out explicitly. +var muslDefaultSearchPath = []string{"/lib", "/usr/local/lib", "/usr/lib"} + +func muslLoader(root, arch string) string { + return filepath.Join(root, "/lib/ld-musl-"+arch+".so.1") +} + +func muslPathFile(root, arch string) string { + return filepath.Join(root, "/etc/ld-musl-"+arch+".path") +} + +// createMuslPathFileIfRequired adds the specified directories to the musl +// .path file in the specified root, which musl searches instead of an ldcache. +func createMuslPathFileIfRequired(root string, driverDirs []string, systemDirs []string) error { + arch, ok := muslArchs[runtime.GOARCH] + if !ok || !isMusl(root, arch) { + return nil + } + + return updateMuslPathFile(muslPathFile(root, arch), driverDirs, systemDirs) +} + +// updateMuslPathFile writes the driver directories that are not searched +// already, then the existing entries, then the system directories to the +// specified .path file. The default search path stands in for a missing file. +// +// The driver directories are searched first so that an injected library wins a +// name lookup against a file of the same name that happens to sit in a +// directory that is searched already. This is the precedence that the glibc +// path gives these directories through the 00-nvcr-*.conf drop-in. +func updateMuslPathFile(path string, driverDirs []string, systemDirs []string) error { + if len(driverDirs) == 0 && len(systemDirs) == 0 { + return nil + } + + existing := muslDefaultSearchPath + if contents, err := os.ReadFile(path); err == nil { + // musl splits the file on colons and newlines. + existing = strings.FieldsFunc(string(contents), func(r rune) bool { return r == ':' || r == '\n' }) + } else if !os.IsNotExist(err) { + return fmt.Errorf("could not read .path file: %w", err) + } + + var dirs []string + for _, dir := range driverDirs { + if !slices.Contains(existing, dir) { + dirs = append(dirs, dir) + } + } + dirs = append(append(dirs, existing...), systemDirs...) + + pathFile, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("could not open .path file: %w", err) + } + defer func() { + _ = pathFile.Close() + }() + + return outputListToFile(pathFile, dirs...) +} + +// isMusl checks whether the container is running musl instead of glibc: its +// dynamic linker is present or, failing that, the container is Alpine-based. +func isMusl(root string, arch string) bool { + return isFile(muslLoader(root, arch)) || isFile(filepath.Join(root, "/etc/alpine-release")) +} + +// isFile checks whether the specified path exists and is not a directory. +func isFile(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} diff --git a/internal/ldconfig/musl_test.go b/internal/ldconfig/musl_test.go new file mode 100644 index 000000000..03a11d531 --- /dev/null +++ b/internal/ldconfig/musl_test.go @@ -0,0 +1,162 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package ldconfig + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreateMuslPathFileIfRequired(t *testing.T) { + arch, ok := muslArchs[runtime.GOARCH] + if !ok { + t.Skip("musl .path files are not handled on this platform") + } + + testCases := []struct { + description string + // isMusl adds the musl dynamic linker. + isMusl bool + // pathFileContents is the contents of the .path file before the update. + pathFileContents *string + driverDirs []string + systemDirs []string + expected *string + }{ + { + description: "glibc container is not modified", + driverDirs: []string{"/driver"}, + systemDirs: []string{"/lib", "/usr/lib"}, + }, + { + description: "path file is created with the default search path preserved", + isMusl: true, + driverDirs: []string{"/driver"}, + systemDirs: []string{"/lib", "/usr/lib"}, + expected: ptr("/driver\n/lib\n/usr/local/lib\n/usr/lib\n"), + }, + { + description: "driver dirs are prepended to the existing contents", + isMusl: true, + pathFileContents: ptr("/lib:/usr/local/lib:/usr/lib"), + driverDirs: []string{"/driver"}, + systemDirs: []string{"/lib", "/usr/lib"}, + expected: ptr("/driver\n/lib\n/usr/local/lib\n/usr/lib\n"), + }, + { + description: "dirs that are searched already are not reordered", + isMusl: true, + pathFileContents: ptr("/lib\n/usr/local/lib\n/driver\n"), + driverDirs: []string{"/driver"}, + expected: ptr("/lib\n/usr/local/lib\n/driver\n"), + }, + { + description: "the order of the driver dirs is maintained", + isMusl: true, + driverDirs: []string{"/driver-2", "/driver-1"}, + expected: ptr("/driver-2\n/driver-1\n/lib\n/usr/local/lib\n/usr/lib\n"), + }, + { + description: "entries separated by colons and newlines are read", + isMusl: true, + pathFileContents: ptr("/lib:/usr/local/lib\n/usr/lib"), + driverDirs: []string{"/driver"}, + expected: ptr("/driver\n/lib\n/usr/local/lib\n/usr/lib\n"), + }, + { + description: "system dirs alone create the path file", + isMusl: true, + systemDirs: []string{"/lib", "/usr/lib"}, + expected: ptr("/lib\n/usr/local/lib\n/usr/lib\n"), + }, + { + description: "no dirs leave the container untouched", + isMusl: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "/etc"), 0o755)) + if tc.pathFileContents != nil { + require.NoError(t, os.WriteFile(muslPathFile(root, arch), []byte(*tc.pathFileContents), 0o600)) + } + if tc.isMusl { + makeMuslLoader(t, root, arch) + } + + require.NoError(t, createMuslPathFileIfRequired(root, tc.driverDirs, tc.systemDirs)) + + requireFileContents(t, muslPathFile(root, arch), tc.expected) + }) + } +} + +func TestIsMusl(t *testing.T) { + arch, ok := muslArchs[runtime.GOARCH] + if !ok { + t.Skip("musl .path files are not handled on this platform") + } + + t.Run("glibc container", func(t *testing.T) { + require.False(t, isMusl(t.TempDir(), arch)) + }) + + t.Run("musl loader", func(t *testing.T) { + root := t.TempDir() + makeMuslLoader(t, root, arch) + require.True(t, isMusl(root, arch)) + }) + + t.Run("alpine release file", func(t *testing.T) { + root := t.TempDir() + makeFile(t, filepath.Join(root, "/etc/alpine-release")) + require.True(t, isMusl(root, arch)) + }) +} + +// makeMuslLoader creates the musl dynamic linker for the specified +// architecture in the specified root. +func makeMuslLoader(t *testing.T, root string, arch string) { + t.Helper() + + makeFile(t, muslLoader(root, arch)) +} + +// requireFileContents checks the contents of the specified file. +// If the expected contents are nil, the file is required to not exist. +func requireFileContents(t *testing.T, path string, expected *string) { + t.Helper() + + contents, err := os.ReadFile(path) + if expected == nil { + require.ErrorIs(t, err, os.ErrNotExist) + return + } + require.NoError(t, err) + require.Equal(t, *expected, string(contents)) +} + +func ptr[T any](v T) *T { + return &v +} diff --git a/internal/lookup/root/options.go b/internal/lookup/root/options.go index e7811b80e..1f6bf2357 100644 --- a/internal/lookup/root/options.go +++ b/internal/lookup/root/options.go @@ -28,7 +28,10 @@ type options struct { librarySearchPaths []string // configSearchPaths specified explicit search paths for discovering driver config files. configSearchPaths []string - versioner Versioner + // compat32 indicates whether the 32-bit driver libraries are also + // discovered. + compat32 bool + versioner Versioner } type Option func(*options) @@ -63,6 +66,14 @@ func WithConfigSearchPaths(paths ...string) Option { } } +// WithCompat32Libraries controls whether the 32-bit driver libraries are also +// discovered. These are discovered by default. +func WithCompat32Libraries(compat32 bool) Option { + return func(o *options) { + o.compat32 = compat32 + } +} + func WithVersioner(versioner Versioner) Option { return func(o *options) { o.versioner = versioner diff --git a/internal/lookup/root/root.go b/internal/lookup/root/root.go index 2866e1d52..1bb4045a8 100644 --- a/internal/lookup/root/root.go +++ b/internal/lookup/root/root.go @@ -41,6 +41,9 @@ type Driver struct { librarySearchPaths []string // configSearchPaths specified explicit search paths for discovering driver config files. configSearchPaths []string + // compat32 indicates whether the 32-bit driver libraries are also + // discovered. + compat32 bool // version caches the driver version. version string @@ -50,7 +53,10 @@ type Driver struct { // New creates a new Driver root using the specified options. func New(opts ...Option) *Driver { - o := &options{} + // The 32-bit driver libraries are discovered unless a caller opts out. + o := &options{ + compat32: true, + } for _, opt := range opts { opt(o) } @@ -76,6 +82,7 @@ func New(opts ...Option) *Driver { DevRoot: o.DevRoot, librarySearchPaths: o.librarySearchPaths, configSearchPaths: o.configSearchPaths, + compat32: o.compat32, version: driverVersion, driverLibDirectories: nil, } @@ -230,6 +237,7 @@ func (r *Driver) Libraries() lookup.Locator { lookup.WithLogger(r.logger), lookup.WithRoot(r.Root), lookup.WithSearchPaths(r.librarySearchPaths...), + lookup.WithCompat32Libraries(r.compat32), ) } diff --git a/internal/oci/state.go b/internal/oci/state.go index 9e6740f0a..4122de2ef 100644 --- a/internal/oci/state.go +++ b/internal/oci/state.go @@ -77,6 +77,19 @@ func (s *State) getRoot() (string, error) { return "", nil } +// GetEnv returns the environment of the container process from the associated +// spec. +func (s *State) GetEnv() ([]string, error) { + spec, err := s.loadMinimalSpec() + if err != nil { + return nil, err + } + if spec.Process == nil { + return nil, nil + } + return spec.Process.Env, nil +} + // GetContainerRoot returns the root for the container from the associated spec. If the spec is not yet loaded, it is // loaded and cached. func (s *State) GetContainerRoot() (string, error) { @@ -114,4 +127,13 @@ func (s *State) loadMinimalSpec() (*minimalSpec, error) { type minimalSpec struct { // Root configures the container's root filesystem. Root *specs.Root `json:"root,omitempty"` + // Process configures the container process. + Process *minimalProcess `json:"process,omitempty"` +} + +// A minimalProcess includes the properties of the container process that are +// required by container lifecycle hooks. +type minimalProcess struct { + // Env is the environment of the container process. + Env []string `json:"env,omitempty"` } diff --git a/internal/oci/state_test.go b/internal/oci/state_test.go index 9a52d0008..62b17be23 100644 --- a/internal/oci/state_test.go +++ b/internal/oci/state_test.go @@ -209,6 +209,58 @@ func TestGetContainerRoot(t *testing.T) { } } +func TestGetEnv(t *testing.T) { + testCases := []struct { + description string + specJSON string + writeSpec bool + isError bool + expectedEnv []string + }{ + { + description: "returns an error when the spec file cannot be loaded", + writeSpec: false, + isError: true, + }, + { + description: "returns nil when the spec has no process", + writeSpec: true, + specJSON: `{}`, + }, + { + description: "returns nil when the process has no environment", + writeSpec: true, + specJSON: `{"process": {}}`, + }, + { + description: "returns the environment of the container process", + writeSpec: true, + specJSON: `{"process": {"env": ["PATH=/usr/bin", "NVIDIA_DRIVER_CAPABILITIES=compute,compat32"]}}`, + expectedEnv: []string{"PATH=/usr/bin", "NVIDIA_DRIVER_CAPABILITIES=compute,compat32"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + dir := t.TempDir() + if tc.writeSpec { + require.NoError(t, os.WriteFile(GetSpecFilePath(dir), []byte(tc.specJSON), 0600)) + } + s := &State{State: specs.State{Bundle: dir}} + + env, err := s.GetEnv() + + if tc.isError { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectedEnv, env) + }) + } +} + func TestLoadMinimalSpec(t *testing.T) { testCases := []struct { description string diff --git a/pkg/lookup/factory.go b/pkg/lookup/factory.go index d3e2acb07..750247e31 100644 --- a/pkg/lookup/factory.go +++ b/pkg/lookup/factory.go @@ -26,12 +26,16 @@ type Factory struct { searchPaths []string filter func(string) error count int + compat32 bool } type Option func(*Factory) func NewFactory(opts ...Option) *Factory { - o := &Factory{} + // The 32-bit libraries on the host are considered unless a caller opts out. + o := &Factory{ + compat32: true, + } for _, opt := range opts { opt(o) } @@ -79,3 +83,11 @@ func WithCount(count int) Option { f.count = count } } + +// WithCompat32Libraries controls whether 32-bit libraries are also considered +// when locating libraries. These are included by default. +func WithCompat32Libraries(compat32 bool) Option { + return func(f *Factory) { + f.compat32 = compat32 + } +} diff --git a/pkg/lookup/ldcache.go b/pkg/lookup/ldcache.go index b39e4adda..577b5622e 100644 --- a/pkg/lookup/ldcache.go +++ b/pkg/lookup/ldcache.go @@ -43,16 +43,28 @@ func (f *Factory) newLdcacheLocator() Locator { f.logger.Warningf("Failed to load ldcache: %v", err) return notFound } + return f.newLdcacheLocatorFrom(cache) +} + +func (f *Factory) newLdcacheLocatorFrom(cache ldcache.LDCache) Locator { + libs32, libs64 := cache.List() + // The 32-bit libraries are searched after the native ones, and only if + // these are not explicitly excluded. + libraryLists := [][]string{libs64} + if f.compat32 { + libraryLists = append(libraryLists, libs32) + } var libraries []string - _, libs64 := cache.List() - for _, library := range libs64 { - chain, err := symlinks.ResolveChain(library) - if err != nil { - f.logger.Warningf("Failed to resolve symlink chain for library %q: %v", library, err) - continue + for _, libs := range libraryLists { + for _, library := range libs { + chain, err := symlinks.ResolveChain(library) + if err != nil { + f.logger.Warningf("Failed to resolve symlink chain for library %q: %v", library, err) + continue + } + libraries = append(libraries, chain...) } - libraries = append(libraries, chain...) } l := &ldcacheLocator{ diff --git a/pkg/lookup/ldcache_test.go b/pkg/lookup/ldcache_test.go index a3a075958..a30b1b4a8 100644 --- a/pkg/lookup/ldcache_test.go +++ b/pkg/lookup/ldcache_test.go @@ -1,12 +1,15 @@ package lookup import ( + "os" "path/filepath" + "strings" "testing" testlog "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + "github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache" "github.com/NVIDIA/nvidia-container-toolkit/internal/test" ) @@ -75,3 +78,53 @@ func TestLDCacheLookup(t *testing.T) { } } } + +func TestLDCacheLookup32BitLibraries(t *testing.T) { + logger, _ := testlog.NewNullLogger() + root := t.TempDir() + + lib64 := filepath.Join(root, "usr/lib64/libcuda.so.999.88.77") + lib32 := filepath.Join(root, "usr/lib/libcuda.so.999.88.77") + require.NoError(t, os.MkdirAll(filepath.Dir(lib64), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Dir(lib32), 0o755)) + require.NoError(t, os.WriteFile(lib64, nil, 0o600)) + require.NoError(t, os.WriteFile(lib32, nil, 0o600)) + + testCases := []struct { + description string + exclude bool + expected []string + }{ + { + description: "32-bit libraries are included after the 64-bit ones", + expected: []string{lib64, lib32}, + }, + { + description: "32-bit libraries can be excluded", + exclude: true, + expected: []string{lib64}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + cache := &ldcache.LDCacheMock{ + ListFunc: func() ([]string, []string) { + return []string{lib32}, []string{lib64} + }, + } + l := NewFactory( + WithLogger(logger), + WithRoot(root), + WithCompat32Libraries(!tc.exclude), + ).newLdcacheLocatorFrom(cache) + + candidates, err := l.Locate("libcuda.so.*") + require.NoError(t, err) + for i := range candidates { + candidates[i] = strings.TrimPrefix(candidates[i], "/private") + } + require.Equal(t, tc.expected, candidates) + }) + } +} diff --git a/pkg/lookup/library.go b/pkg/lookup/library.go index e57bd0e17..a60a3bb4d 100644 --- a/pkg/lookup/library.go +++ b/pkg/lookup/library.go @@ -19,11 +19,17 @@ package lookup // NewLibraryLocator creates a library locator using the specified options. // If search paths (WithSearchPaths(path1, path2, ...)) are explicitly specified // a library locator using these as absolute paths are used. Otherwise the -// library is constructed using the following ordering, returning the first -// successful result: -// - attempt to locate the library / pattern using dlopen -// - attempt to locate the library from a set of predefined search paths. -// - attempt to locate the library from the ldcache. +// library locator combines the unique matches from the following sources, in +// precedence order: +// - a set of predefined search paths +// - the 64-bit entries of the ldcache +// - the 32-bit entries of the ldcache +// +// A 32-bit library is typically in a directory that is not in the predefined +// search paths, so the ldcache is consulted even if one of these already +// provided a match. If 32-bit libraries are excluded +// (WithCompat32Libraries(false)), the first source with a match is used +// instead. func NewLibraryLocator(opts ...Option) Locator { f := NewFactory(opts...) @@ -50,9 +56,15 @@ func NewLibraryLocator(opts ...Option) Locator { "/lib/aarch64-linux-gnu/nvidia/current", }...), ) - l := First( + if !f.compat32 { + return First( + NewSymlinkLocator(opts...), + f.newLdcacheLocator(), + ) + } + + return AsUnique(Merge( NewSymlinkLocator(opts...), f.newLdcacheLocator(), - ) - return l + )) } diff --git a/pkg/lookup/library_test.go b/pkg/lookup/library_test.go index 8837ae99a..2968a3289 100644 --- a/pkg/lookup/library_test.go +++ b/pkg/lookup/library_test.go @@ -24,6 +24,8 @@ import ( testlog "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/test" ) func TestLibraryLocator(t *testing.T) { @@ -136,3 +138,64 @@ func TestLibraryLocator(t *testing.T) { }) } } + +func TestLibraryLocatorCompat32(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + moduleRoot, err := test.GetModuleRoot() + require.NoError(t, err) + + // We construct a root with a library in one of the predefined search paths + // and an ldcache -- from the rootfs-2 testdata -- that instead refers to + // libraries in /var/lib/nvidia/lib64. + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "/etc"), 0o755)) + require.NoError(t, os.Symlink( + filepath.Join(moduleRoot, "testdata/lookup/rootfs-2/etc/ld.so.cache"), + filepath.Join(root, "/etc/ld.so.cache"), + )) + + inSearchPath := filepath.Join(root, "/usr/lib64/libcuda.so.999.88.77") + inLdcache := filepath.Join(root, "/var/lib/nvidia/lib64/libcuda.so.999.88.77") + for _, library := range []string{inSearchPath, inLdcache} { + require.NoError(t, os.MkdirAll(filepath.Dir(library), 0o755)) + require.NoError(t, os.WriteFile(library, nil, 0o600)) + require.NoError(t, os.Symlink(library, filepath.Join(filepath.Dir(library), "libcuda.so.1"))) + } + + testCases := []struct { + description string + exclude bool + expected []string + }{ + { + description: "the ldcache is also consulted for 32-bit libraries", + expected: []string{inSearchPath, inLdcache}, + }, + { + description: "the ldcache is not consulted if a search path matches and 32-bit libraries are excluded", + exclude: true, + expected: []string{inSearchPath}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + lut := NewLibraryLocator( + WithLogger(logger), + WithRoot(root), + WithCompat32Libraries(!tc.exclude), + ) + + candidates, err := lut.Locate("libcuda.so.1") + require.NoError(t, err) + + var cleanedCandidates []string + for _, c := range candidates { + // On MacOS /var and /tmp symlink to /private/var and /private/tmp which is included in the resolved path. + cleanedCandidates = append(cleanedCandidates, strings.TrimPrefix(c, "/private")) + } + require.EqualValues(t, tc.expected, cleanedCandidates) + }) + } +} diff --git a/pkg/lookup/merge.go b/pkg/lookup/merge.go index ade3dd5ae..a2cd8c2a8 100644 --- a/pkg/lookup/merge.go +++ b/pkg/lookup/merge.go @@ -21,6 +21,7 @@ import ( ) type first []Locator +type merged []Locator type unique struct { locator Locator @@ -38,6 +39,21 @@ func First(locators ...Locator) Locator { return f } +// Merge returns a locator that combines matches from all supplied locators in +// argument order. Nil locators are ignored. If at least one locator returns a +// match, errors from the other locators are ignored; otherwise, all locator +// errors are joined and returned. +func Merge(locators ...Locator) Locator { + var m merged + for _, l := range locators { + if l == nil { + continue + } + m = append(m, l) + } + return m +} + // Locate returns the results for the first locator that returns a non-empty non-error result. func (f first) Locate(pattern string) ([]string, error) { var allErrors []error @@ -58,6 +74,26 @@ func (f first) Locate(pattern string) ([]string, error) { return nil, errors.Join(allErrors...) } +// Locate returns the combined results from all locators that return matches. +func (m merged) Locate(pattern string) ([]string, error) { + var candidates []string + var allErrors []error + for _, l := range m { + matches, err := l.Locate(pattern) + if err != nil { + allErrors = append(allErrors, err) + continue + } + candidates = append(candidates, matches...) + } + + if len(candidates) > 0 { + return candidates, nil + } + + return nil, errors.Join(allErrors...) +} + func AsUnique(locator Locator) Locator { return &unique{ locator: locator, diff --git a/pkg/lookup/merge_test.go b/pkg/lookup/merge_test.go new file mode 100644 index 000000000..c81daf63d --- /dev/null +++ b/pkg/lookup/merge_test.go @@ -0,0 +1,58 @@ +/** +# Copyright 2026 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package lookup + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMerge(t *testing.T) { + first := &LocatorMock{ + LocateFunc: func(string) ([]string, error) { + return []string{"first"}, nil + }, + } + second := &LocatorMock{ + LocateFunc: func(string) ([]string, error) { + return []string{"second"}, nil + }, + } + + candidates, err := Merge(first, second).Locate("libcuda.so.*") + require.NoError(t, err) + require.Equal(t, []string{"first", "second"}, candidates) +} + +func TestMergeReturnsMatchesWhenAnotherLocatorFails(t *testing.T) { + failing := &LocatorMock{ + LocateFunc: func(string) ([]string, error) { + return nil, fmt.Errorf("lookup failed") + }, + } + matching := &LocatorMock{ + LocateFunc: func(string) ([]string, error) { + return []string{"match"}, nil + }, + } + + candidates, err := Merge(failing, matching).Locate("libcuda.so.*") + require.NoError(t, err) + require.Equal(t, []string{"match"}, candidates) +} diff --git a/pkg/nvcdi/api.go b/pkg/nvcdi/api.go index 7bf2fdc12..2f8ba6e8c 100644 --- a/pkg/nvcdi/api.go +++ b/pkg/nvcdi/api.go @@ -100,4 +100,10 @@ const ( // FeatureDisableIPCDiscoverer disables the inclusion of IPC sockets // (nvidia-persistenced, nvidia-fabricmanager, MPS) in the CDI spec. FeatureDisableIPCDiscoverer = FeatureFlag("disable-ipc-discoverer") + + // FeatureDisableCompat32Libraries disables the inclusion of the 32-bit + // compatibility driver libraries installed on the host. + // These are included by default and are only exposed to a container that + // can run 32-bit applications. See the update-ldcache hook. + FeatureDisableCompat32Libraries = FeatureFlag("disable-compat32-libraries") ) diff --git a/pkg/nvcdi/lib.go b/pkg/nvcdi/lib.go index fa84e19c5..6e846a3a0 100644 --- a/pkg/nvcdi/lib.go +++ b/pkg/nvcdi/lib.go @@ -222,6 +222,7 @@ func (o *options) getDriverOptions() []root.Option { root.WithDevRoot(o.devRoot), root.WithLibrarySearchPaths(o.librarySearchPaths...), root.WithConfigSearchPaths(o.configSearchPaths...), + root.WithCompat32Libraries(!o.featureFlags[FeatureDisableCompat32Libraries]), root.WithVersioner( root.FirstOf( nvsandboxutilslibWithVersion(o.nvsandboxutilslib), diff --git a/pkg/nvcdi/options.go b/pkg/nvcdi/options.go index 111ae6981..38563b08e 100644 --- a/pkg/nvcdi/options.go +++ b/pkg/nvcdi/options.go @@ -133,6 +133,7 @@ func (o *options) driverLibraryLocator() lookup.Locator { lookup.WithLogger(o.logger), lookup.WithRoot(o.driverRoot), lookup.WithSearchPaths(o.librarySearchPaths...), + lookup.WithCompat32Libraries(!o.featureFlags[FeatureDisableCompat32Libraries]), ) }