diff --git a/api/config/v1/runtime.go b/api/config/v1/runtime.go index 6f58f3764..447bbfa0e 100644 --- a/api/config/v1/runtime.go +++ b/api/config/v1/runtime.go @@ -27,6 +27,15 @@ type RuntimeConfig struct { Runtimes []string `toml:"runtimes"` Mode string `toml:"mode"` Modes modesConfig `toml:"modes"` + // Rlimits defines POSIX resource limits to apply to the OCI runtime + // specification of containers handled by the NVIDIA Container Runtime. + // Each entry has the form NAME=SOFT[:HARD], where NAME is an rlimit name + // with or without the RLIMIT_ prefix (case-insensitive, e.g. "memlock" or + // "RLIMIT_MEMLOCK") and the values are non-negative integers or + // "unlimited"/"infinity". If HARD is omitted, it is set to SOFT. + // A configured entry replaces an rlimit of the same type already present + // in the incoming OCI specification. + Rlimits []string `toml:"rlimits,omitempty"` } // modesConfig defines (optional) per-mode configs diff --git a/cmd/nvidia-ctk-installer/toolkit/toolkit.go b/cmd/nvidia-ctk-installer/toolkit/toolkit.go index ebfa0a557..840336705 100644 --- a/cmd/nvidia-ctk-installer/toolkit/toolkit.go +++ b/cmd/nvidia-ctk-installer/toolkit/toolkit.go @@ -63,6 +63,8 @@ type Options struct { ContainerRuntimeRuntimes []string + ContainerRuntimeRlimits []string + ContainerRuntimeHookSkipModeDetection bool ContainerCLIDebug string @@ -141,6 +143,12 @@ func Flags(opts *Options) []cli.Flag { Destination: &opts.ContainerRuntimeRuntimes, Sources: cli.EnvVars("NVIDIA_CONTAINER_RUNTIME_RUNTIMES"), }, + &cli.StringSliceFlag{ + Name: "nvidia-container-runtime.rlimits", + Usage: "Specify POSIX rlimits (e.g. memlock=unlimited) that the NVIDIA Container Runtime applies to the containers it handles", + Destination: &opts.ContainerRuntimeRlimits, + Sources: cli.EnvVars("NVIDIA_CONTAINER_RUNTIME_RLIMITS"), + }, &cli.BoolFlag{ Name: "nvidia-container-runtime-hook.skip-mode-detection", Value: true, @@ -444,6 +452,7 @@ func (t *Installer) installToolkitConfig(c *cli.Command, opts *Options) error { "nvidia-container-runtime.modes.cdi.annotation-prefixes": opts.ContainerRuntimeModesCDIAnnotationPrefixes, "nvidia-container-runtime.modes.cdi.default-kind": opts.ContainerRuntimeModesCdiDefaultKind, "nvidia-container-runtime.runtimes": opts.ContainerRuntimeRuntimes, + "nvidia-container-runtime.rlimits": opts.ContainerRuntimeRlimits, "nvidia-container-cli.debug": opts.ContainerCLIDebug, } @@ -467,6 +476,10 @@ func (t *Installer) installToolkitConfig(c *cli.Command, opts *Options) error { continue } value = v.Value() + case []string: + if len(v) == 0 { + continue + } default: t.logger.Warningf("Unexpected type for option %v=%v: %T", key, value, v) } diff --git a/internal/modifier/factory.go b/internal/modifier/factory.go index 3ec104fa2..e3a37bcf1 100644 --- a/internal/modifier/factory.go +++ b/internal/modifier/factory.go @@ -126,6 +126,12 @@ func (f *Factory) create() (oci.SpecModifier, error) { return nil, err } modifiers = append(modifiers, featureGatedModifier) + case "rlimits": + rlimitModifier, err := f.newRlimitModifier() + if err != nil { + return nil, err + } + modifiers = append(modifiers, rlimitModifier) default: f.logger.Debugf("Ignoring unknown modifier type %q", modifierType) } diff --git a/internal/modifier/mode.go b/internal/modifier/mode.go index f1abd9c13..aa0307163 100644 --- a/internal/modifier/mode.go +++ b/internal/modifier/mode.go @@ -21,15 +21,17 @@ func (f *Factory) newModeModifier() (oci.SpecModifier, error) { } // supportedModifierTypes returns the modifiers supported for a specific runtime mode. +// The rlimits modifier applies in every mode: it is driven purely by the +// runtime config and is independent of how devices are injected. func supportedModifierTypes(mode info.RuntimeMode) []string { switch mode { case info.CDIRuntimeMode, info.JitCDIRuntimeMode: - // For CDI mode we make no additional modifications. - return []string{"nvidia-hook-remover", "mode"} + // For CDI mode we make no additional device modifications. + return []string{"nvidia-hook-remover", "mode", "rlimits"} case info.CSVRuntimeMode: // For CSV mode we support mode and feature-gated modification. - return []string{"nvidia-hook-remover", "feature-gated", "mode"} + return []string{"nvidia-hook-remover", "feature-gated", "mode", "rlimits"} default: - return []string{"feature-gated", "graphics", "mode"} + return []string{"feature-gated", "graphics", "mode", "rlimits"} } } diff --git a/internal/modifier/rlimit.go b/internal/modifier/rlimit.go new file mode 100644 index 000000000..37b140333 --- /dev/null +++ b/internal/modifier/rlimit.go @@ -0,0 +1,163 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package modifier + +import ( + "fmt" + "math" + "strconv" + "strings" + + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" +) + +// newRlimitModifier constructs a modifier that applies the POSIX rlimits +// configured for the NVIDIA Container Runtime to the OCI runtime +// specification. This allows resource limits such as RLIMIT_MEMLOCK -- which +// RDMA workloads require far in excess of typical daemon defaults -- to be +// set for the containers handled by this runtime without changing the limits +// of every container on the host. +// +// The modifier is intentionally not gated on requested devices: workloads +// that need the configured limits (e.g. RDMA-only containers) do not +// necessarily request GPUs. +func (f *Factory) newRlimitModifier() (oci.SpecModifier, error) { + rlimits, err := parseRlimits(f.cfg.NVIDIAContainerRuntimeConfig.Rlimits) + if err != nil { + return nil, fmt.Errorf("failed to parse rlimits: %w", err) + } + if len(rlimits) == 0 { + return nil, nil + } + return &rlimitModifier{ + logger: f.logger, + rlimits: rlimits, + }, nil +} + +type rlimitModifier struct { + logger logger.Interface + rlimits []specs.POSIXRlimit +} + +var _ oci.SpecModifier = (*rlimitModifier)(nil) + +// Modify applies the configured rlimits to the OCI spec. An entry replaces an +// existing rlimit of the same type; other entries in the spec are preserved. +func (m *rlimitModifier) Modify(spec *specs.Spec) error { + if spec == nil { + return fmt.Errorf("cannot modify nil spec") + } + if spec.Process == nil { + spec.Process = &specs.Process{} + } + for _, rlimit := range m.rlimits { + m.logger.Debugf("Setting rlimit %v to %v:%v", rlimit.Type, rlimit.Soft, rlimit.Hard) + spec.Process.Rlimits = upsertRlimit(spec.Process.Rlimits, rlimit) + } + return nil +} + +// upsertRlimit replaces the rlimit of the same type in the supplied list, or +// appends it if no such entry exists. +func upsertRlimit(rlimits []specs.POSIXRlimit, rlimit specs.POSIXRlimit) []specs.POSIXRlimit { + for i, existing := range rlimits { + if existing.Type == rlimit.Type { + rlimits[i] = rlimit + return rlimits + } + } + return append(rlimits, rlimit) +} + +// parseRlimits converts configured NAME=SOFT[:HARD] entries into OCI POSIX +// rlimits. Duplicate types are rejected to surface configuration mistakes +// instead of silently applying one of the values. +func parseRlimits(entries []string) ([]specs.POSIXRlimit, error) { + var rlimits []specs.POSIXRlimit + seen := make(map[string]bool) + for _, entry := range entries { + rlimit, err := parseRlimit(entry) + if err != nil { + return nil, err + } + if seen[rlimit.Type] { + return nil, fmt.Errorf("duplicate rlimit type %v", rlimit.Type) + } + seen[rlimit.Type] = true + rlimits = append(rlimits, rlimit) + } + return rlimits, nil +} + +// parseRlimit parses a single NAME=SOFT[:HARD] entry. The name is +// case-insensitive and the RLIMIT_ prefix is optional; values are +// non-negative integers or "unlimited"/"infinity". If HARD is omitted, it is +// set to SOFT. +func parseRlimit(entry string) (specs.POSIXRlimit, error) { + name, values, ok := strings.Cut(entry, "=") + name = strings.TrimSpace(name) + if !ok || name == "" { + return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: expected NAME=SOFT[:HARD]", entry) + } + + rlimitType := strings.ToUpper(name) + if !strings.HasPrefix(rlimitType, "RLIMIT_") { + rlimitType = "RLIMIT_" + rlimitType + } + + softValue, hardValue, ok := strings.Cut(values, ":") + if !ok { + hardValue = softValue + } + soft, err := parseRlimitValue(softValue) + if err != nil { + return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: %w", entry, err) + } + hard, err := parseRlimitValue(hardValue) + if err != nil { + return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: %w", entry, err) + } + if soft > hard { + return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: soft limit %v exceeds hard limit %v", entry, softValue, hardValue) + } + + return specs.POSIXRlimit{ + Type: rlimitType, + Soft: soft, + Hard: hard, + }, nil +} + +// parseRlimitValue parses a single rlimit value: a non-negative integer or +// "unlimited"/"infinity", which map to RLIM_INFINITY. +func parseRlimitValue(value string) (uint64, error) { + value = strings.TrimSpace(value) + switch strings.ToLower(value) { + case "unlimited", "infinity": + return math.MaxUint64, nil + } + parsed, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid value %q: expected a non-negative integer, \"unlimited\", or \"infinity\"", value) + } + return parsed, nil +} diff --git a/internal/modifier/rlimit_test.go b/internal/modifier/rlimit_test.go new file mode 100644 index 000000000..e28068969 --- /dev/null +++ b/internal/modifier/rlimit_test.go @@ -0,0 +1,243 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package modifier + +import ( + "math" + "testing" + + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/NVIDIA/nvidia-container-toolkit/api/config/v1" +) + +func TestParseRlimits(t *testing.T) { + testCases := []struct { + description string + entries []string + expected []specs.POSIXRlimit + expectedError bool + }{ + { + description: "empty entries yield no rlimits", + }, + { + description: "unlimited value applies to soft and hard", + entries: []string{"memlock=unlimited"}, + expected: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}, + }, + }, + { + description: "infinity is accepted as an alias", + entries: []string{"memlock=infinity"}, + expected: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}, + }, + }, + { + description: "RLIMIT_ prefix and case are normalized", + entries: []string{"RLIMIT_MEMLOCK=unlimited", "NoFile=1024:2048"}, + expected: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}, + {Type: "RLIMIT_NOFILE", Soft: 1024, Hard: 2048}, + }, + }, + { + description: "numeric soft and hard values", + entries: []string{"memlock=1073741824:2147483648"}, + expected: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: 1073741824, Hard: 2147483648}, + }, + }, + { + description: "numeric soft with unlimited hard", + entries: []string{"memlock=1073741824:unlimited"}, + expected: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: 1073741824, Hard: math.MaxUint64}, + }, + }, + { + description: "missing separator is rejected", + entries: []string{"memlock"}, + expectedError: true, + }, + { + description: "empty name is rejected", + entries: []string{"=unlimited"}, + expectedError: true, + }, + { + description: "non-numeric value is rejected", + entries: []string{"memlock=lots"}, + expectedError: true, + }, + { + description: "negative value is rejected", + entries: []string{"memlock=-1"}, + expectedError: true, + }, + { + description: "soft limit above hard limit is rejected", + entries: []string{"memlock=2048:1024"}, + expectedError: true, + }, + { + description: "duplicate types are rejected", + entries: []string{"memlock=unlimited", "RLIMIT_MEMLOCK=1024"}, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + rlimits, err := parseRlimits(tc.entries) + if tc.expectedError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, tc.expected, rlimits) + }) + } +} + +func TestRlimitModifier(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + testCases := []struct { + description string + rlimits []specs.POSIXRlimit + spec *specs.Spec + expectedError bool + expectedLimits []specs.POSIXRlimit + }{ + { + description: "nil spec is rejected", + rlimits: []specs.POSIXRlimit{{Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}}, + expectedError: true, + }, + { + description: "nil process is initialized", + rlimits: []specs.POSIXRlimit{{Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}}, + spec: &specs.Spec{}, + expectedLimits: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}, + }, + }, + { + description: "existing rlimit of the same type is replaced", + rlimits: []specs.POSIXRlimit{{Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}}, + spec: &specs.Spec{ + Process: &specs.Process{ + Rlimits: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: 65536, Hard: 65536}, + {Type: "RLIMIT_NOFILE", Soft: 1024, Hard: 1024}, + }, + }, + }, + expectedLimits: []specs.POSIXRlimit{ + {Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}, + {Type: "RLIMIT_NOFILE", Soft: 1024, Hard: 1024}, + }, + }, + { + description: "rlimits of other types are appended", + rlimits: []specs.POSIXRlimit{{Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}}, + spec: &specs.Spec{ + Process: &specs.Process{ + Rlimits: []specs.POSIXRlimit{ + {Type: "RLIMIT_NOFILE", Soft: 1024, Hard: 1024}, + }, + }, + }, + expectedLimits: []specs.POSIXRlimit{ + {Type: "RLIMIT_NOFILE", Soft: 1024, Hard: 1024}, + {Type: "RLIMIT_MEMLOCK", Soft: math.MaxUint64, Hard: math.MaxUint64}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + m := rlimitModifier{ + logger: logger, + rlimits: tc.rlimits, + } + err := m.Modify(tc.spec) + if tc.expectedError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, tc.expectedLimits, tc.spec.Process.Rlimits) + }) + } +} + +func TestNewRlimitModifier(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + testCases := []struct { + description string + rlimits []string + expectedError bool + expectedModifier bool + }{ + { + description: "no configured rlimits yield no modifier", + }, + { + description: "configured rlimits yield a modifier", + rlimits: []string{"memlock=unlimited"}, + expectedModifier: true, + }, + { + description: "invalid rlimits raise an error", + rlimits: []string{"memlock=lots"}, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + f := createFactory( + WithLogger(logger), + WithConfig(&config.Config{ + NVIDIAContainerRuntimeConfig: config.RuntimeConfig{ + Rlimits: tc.rlimits, + }, + }), + ) + m, err := f.newRlimitModifier() + if tc.expectedError { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.expectedModifier { + require.NotNil(t, m) + } else { + require.Nil(t, m) + } + }) + } +}