Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions api/config/v1/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@ type RuntimeConfig struct {
Runtimes []string `toml:"runtimes"`
Mode string `toml:"mode"`
Modes modesConfig `toml:"modes"`
// Rlimits defines POSIX resource limits to apply to the OCI runtime
// specification of containers handled by the NVIDIA Container Runtime.
// Each entry has the form NAME=SOFT[:HARD], where NAME is an rlimit name
// with or without the RLIMIT_ prefix (case-insensitive, e.g. "memlock" or
// "RLIMIT_MEMLOCK") and the values are non-negative integers or
// "unlimited"/"infinity". If HARD is omitted, it is set to SOFT.
// A configured entry replaces an rlimit of the same type already present
// in the incoming OCI specification.
Rlimits []string `toml:"rlimits,omitempty"`
}

// modesConfig defines (optional) per-mode configs
Expand Down
13 changes: 13 additions & 0 deletions cmd/nvidia-ctk-installer/toolkit/toolkit.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ type Options struct {

ContainerRuntimeRuntimes []string

ContainerRuntimeRlimits []string

ContainerRuntimeHookSkipModeDetection bool

ContainerCLIDebug string
Expand Down Expand Up @@ -141,6 +143,12 @@ func Flags(opts *Options) []cli.Flag {
Destination: &opts.ContainerRuntimeRuntimes,
Sources: cli.EnvVars("NVIDIA_CONTAINER_RUNTIME_RUNTIMES"),
},
&cli.StringSliceFlag{
Name: "nvidia-container-runtime.rlimits",
Usage: "Specify POSIX rlimits (e.g. memlock=unlimited) that the NVIDIA Container Runtime applies to the containers it handles",
Destination: &opts.ContainerRuntimeRlimits,
Sources: cli.EnvVars("NVIDIA_CONTAINER_RUNTIME_RLIMITS"),
},
&cli.BoolFlag{
Name: "nvidia-container-runtime-hook.skip-mode-detection",
Value: true,
Expand Down Expand Up @@ -444,6 +452,7 @@ func (t *Installer) installToolkitConfig(c *cli.Command, opts *Options) error {
"nvidia-container-runtime.modes.cdi.annotation-prefixes": opts.ContainerRuntimeModesCDIAnnotationPrefixes,
"nvidia-container-runtime.modes.cdi.default-kind": opts.ContainerRuntimeModesCdiDefaultKind,
"nvidia-container-runtime.runtimes": opts.ContainerRuntimeRuntimes,
"nvidia-container-runtime.rlimits": opts.ContainerRuntimeRlimits,
"nvidia-container-cli.debug": opts.ContainerCLIDebug,
}

Expand All @@ -467,6 +476,10 @@ func (t *Installer) installToolkitConfig(c *cli.Command, opts *Options) error {
continue
}
value = v.Value()
case []string:
if len(v) == 0 {
continue
}
default:
t.logger.Warningf("Unexpected type for option %v=%v: %T", key, value, v)
}
Expand Down
6 changes: 6 additions & 0 deletions internal/modifier/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ func (f *Factory) create() (oci.SpecModifier, error) {
return nil, err
}
modifiers = append(modifiers, featureGatedModifier)
case "rlimits":
rlimitModifier, err := f.newRlimitModifier()
if err != nil {
return nil, err
}
modifiers = append(modifiers, rlimitModifier)
default:
f.logger.Debugf("Ignoring unknown modifier type %q", modifierType)
}
Expand Down
10 changes: 6 additions & 4 deletions internal/modifier/mode.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,17 @@ func (f *Factory) newModeModifier() (oci.SpecModifier, error) {
}

// supportedModifierTypes returns the modifiers supported for a specific runtime mode.
// The rlimits modifier applies in every mode: it is driven purely by the
// runtime config and is independent of how devices are injected.
func supportedModifierTypes(mode info.RuntimeMode) []string {
switch mode {
case info.CDIRuntimeMode, info.JitCDIRuntimeMode:
// For CDI mode we make no additional modifications.
return []string{"nvidia-hook-remover", "mode"}
// For CDI mode we make no additional device modifications.
return []string{"nvidia-hook-remover", "mode", "rlimits"}
case info.CSVRuntimeMode:
// For CSV mode we support mode and feature-gated modification.
return []string{"nvidia-hook-remover", "feature-gated", "mode"}
return []string{"nvidia-hook-remover", "feature-gated", "mode", "rlimits"}
default:
return []string{"feature-gated", "graphics", "mode"}
return []string{"feature-gated", "graphics", "mode", "rlimits"}
}
}
163 changes: 163 additions & 0 deletions internal/modifier/rlimit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
/**
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

package modifier

import (
"fmt"
"math"
"strconv"
"strings"

"github.com/opencontainers/runtime-spec/specs-go"

"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
)

// newRlimitModifier constructs a modifier that applies the POSIX rlimits
// configured for the NVIDIA Container Runtime to the OCI runtime
// specification. This allows resource limits such as RLIMIT_MEMLOCK -- which
// RDMA workloads require far in excess of typical daemon defaults -- to be
// set for the containers handled by this runtime without changing the limits
// of every container on the host.
//
// The modifier is intentionally not gated on requested devices: workloads
// that need the configured limits (e.g. RDMA-only containers) do not
// necessarily request GPUs.
func (f *Factory) newRlimitModifier() (oci.SpecModifier, error) {
rlimits, err := parseRlimits(f.cfg.NVIDIAContainerRuntimeConfig.Rlimits)
if err != nil {
return nil, fmt.Errorf("failed to parse rlimits: %w", err)
}
if len(rlimits) == 0 {
return nil, nil
}
return &rlimitModifier{
logger: f.logger,
rlimits: rlimits,
}, nil
}

type rlimitModifier struct {
logger logger.Interface
rlimits []specs.POSIXRlimit
}

var _ oci.SpecModifier = (*rlimitModifier)(nil)

// Modify applies the configured rlimits to the OCI spec. An entry replaces an
// existing rlimit of the same type; other entries in the spec are preserved.
func (m *rlimitModifier) Modify(spec *specs.Spec) error {
if spec == nil {
return fmt.Errorf("cannot modify nil spec")
}
if spec.Process == nil {
spec.Process = &specs.Process{}
}
for _, rlimit := range m.rlimits {
m.logger.Debugf("Setting rlimit %v to %v:%v", rlimit.Type, rlimit.Soft, rlimit.Hard)
spec.Process.Rlimits = upsertRlimit(spec.Process.Rlimits, rlimit)
}
return nil
}

// upsertRlimit replaces the rlimit of the same type in the supplied list, or
// appends it if no such entry exists.
func upsertRlimit(rlimits []specs.POSIXRlimit, rlimit specs.POSIXRlimit) []specs.POSIXRlimit {
for i, existing := range rlimits {
if existing.Type == rlimit.Type {
rlimits[i] = rlimit
return rlimits
}
}
return append(rlimits, rlimit)
}

// parseRlimits converts configured NAME=SOFT[:HARD] entries into OCI POSIX
// rlimits. Duplicate types are rejected to surface configuration mistakes
// instead of silently applying one of the values.
func parseRlimits(entries []string) ([]specs.POSIXRlimit, error) {
var rlimits []specs.POSIXRlimit
seen := make(map[string]bool)
for _, entry := range entries {
rlimit, err := parseRlimit(entry)
if err != nil {
return nil, err
}
if seen[rlimit.Type] {
return nil, fmt.Errorf("duplicate rlimit type %v", rlimit.Type)
}
seen[rlimit.Type] = true
rlimits = append(rlimits, rlimit)
}
return rlimits, nil
}

// parseRlimit parses a single NAME=SOFT[:HARD] entry. The name is
// case-insensitive and the RLIMIT_ prefix is optional; values are
// non-negative integers or "unlimited"/"infinity". If HARD is omitted, it is
// set to SOFT.
func parseRlimit(entry string) (specs.POSIXRlimit, error) {
name, values, ok := strings.Cut(entry, "=")
name = strings.TrimSpace(name)
if !ok || name == "" {
return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: expected NAME=SOFT[:HARD]", entry)
}

rlimitType := strings.ToUpper(name)
if !strings.HasPrefix(rlimitType, "RLIMIT_") {
rlimitType = "RLIMIT_" + rlimitType
}

softValue, hardValue, ok := strings.Cut(values, ":")
if !ok {
hardValue = softValue
}
soft, err := parseRlimitValue(softValue)
if err != nil {
return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: %w", entry, err)
}
hard, err := parseRlimitValue(hardValue)
if err != nil {
return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: %w", entry, err)
}
if soft > hard {
return specs.POSIXRlimit{}, fmt.Errorf("invalid rlimit %q: soft limit %v exceeds hard limit %v", entry, softValue, hardValue)
}

return specs.POSIXRlimit{
Type: rlimitType,
Soft: soft,
Hard: hard,
}, nil
}

// parseRlimitValue parses a single rlimit value: a non-negative integer or
// "unlimited"/"infinity", which map to RLIM_INFINITY.
func parseRlimitValue(value string) (uint64, error) {
value = strings.TrimSpace(value)
switch strings.ToLower(value) {
case "unlimited", "infinity":
return math.MaxUint64, nil
}
parsed, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid value %q: expected a non-negative integer, \"unlimited\", or \"infinity\"", value)
}
return parsed, nil
}
Loading