Support recursive environment variable fallback in .yarnrc.yml

.yarn/versions/512a4c5a.yml

@@ -0,0 +1,36 @@
+releases:
+  "@yarnpkg/cli": patch
+  "@yarnpkg/core": patch
+
+declined:
+  - "@yarnpkg/plugin-catalog"
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-jsr"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

packages/docusaurus/static/configuration/yarnrc.json

@@ -1,7 +1,7 @@
 {
   "title": "JSON Schema for Yarnrc files",
   "$schema": "https://json-schema.org/draft/2019-09/schema#",
-  "description": "Yarnrc files (named this way because they must be called `.yarnrc.yml`) are the one place where you'll be able to configure Yarn's internal settings. While Yarn will automatically find them in the parent directories, they should usually be kept at the root of your project (often your repository). **Starting from the v2, they must be written in valid Yaml and have the right extension** (simply calling your file `.yarnrc` won't do).\n\nEnvironment variables can be accessed from setting definitions by using the `${NAME}` syntax when defining the values. By default Yarn will require the variables to be present, but this can be turned off by using either `${NAME-fallback}` (which will return `fallback` if `NAME` isn't set) or `${NAME:-fallback}` (which will return `fallback` if `NAME` isn't set, or is an empty string).\n\nFinally, note that most settings can also be defined through environment variables (at least for the simpler ones; arrays and objects aren't supported yet). To do this, just prefix the names and write them in snake case: `YARN_CACHE_FOLDER` will set the cache folder (such values will overwrite any that might have been defined in the RC files - use them sparingly).",
+  "description": "Yarnrc files (named this way because they must be called `.yarnrc.yml`) are the one place where you'll be able to configure Yarn's internal settings. While Yarn will automatically find them in the parent directories, they should usually be kept at the root of your project (often your repository). **Starting from the v2, they must be written in valid Yaml and have the right extension** (simply calling your file `.yarnrc` won't do).\n\nEnvironment variables can be accessed from setting definitions by using the `${NAME}` syntax when defining the values. By default Yarn will require the variables to be present, but this can be turned off by using either `${NAME-fallback}` (which will return `fallback` if `NAME` isn't set) or `${NAME:-fallback}` (which will return `fallback` if `NAME` isn't set, or is an empty string). You can also nest this syntax, for example `${PRIMARY:-${SECONDARY:-fallback}}`, which first checks PRIMARY, then SECONDARY, and finally falls back to fallback.\n\nFinally, note that most settings can also be defined through environment variables (at least for the simpler ones; arrays and objects aren't supported yet). To do this, just prefix the names and write them in snake case: `YARN_CACHE_FOLDER` will set the cache folder (such values will overwrite any that might have been defined in the RC files - use them sparingly).",
   "__info": [
     "This file contains the JSON Schema for Yarnrc files and is:",
     "1) Hosted on the Yarn Website at http://yarnpkg.com/configuration/yarnrc.json",

packages/yarnpkg-core/sources/miscUtils.ts

@@ -471,9 +471,9 @@ export function buildIgnorePattern(ignorePatterns: Array<string>) {
 }
 
 export function replaceEnvVariables(value: string, {env}: {env: {[key: string]: string | undefined}}) {
-  const regex = /\\?\${(?<variableName>[\d\w_]+)(?<colon>:)?(?:-(?<fallback>[^}]*))?}/g;
+  const regex = /\\?\${(?<variableName>[\d\w_]+)(?<colon>:)?(?:-(?<fallback>(?:[^}]|}(?=}))*))?}/g;
 
-  return value.replace(regex, (match, ...args) => {
+  return value.replace(regex, (match: string, ...args: Array<any>) => {
     if (match.startsWith(`\\`))
       return match.slice(1);
 
@@ -486,7 +486,7 @@ export function replaceEnvVariables(value: string, {env}: {env: {[key: string]:
     if (variableExist && !colon)
       return variableValue;
     if (fallback != null)
-      return fallback;
+      return replaceEnvVariables(fallback, {env});
 
     throw new UsageError(`Environment variable not found (${variableName})`);
   });

packages/yarnpkg-core/tests/Configuration.test.ts

@@ -114,6 +114,12 @@ describe(`Configuration`, () => {
           emptyEnvWithEmptyFallback: {
             npmAuthToken: `\${EMPTY_VARIABLE:-}`,
           },
+          emptyEnvWithNestedEnv: {
+            npmAuthToken: `\${EMPTY_VARIABLE:-\${ENV_AUTH_TOKEN:-fallback-value}}`,
+          },
+          multipleNestedEmptyEnvWithFallback: {
+            npmAuthToken: `\${EMPTY_VARIABLE:-\${EMPTY_VARIABLE:-fallback-value}}`,
+          },
         },
       }, async dir => {
         const configuration = await Configuration.find(dir, {
@@ -132,6 +138,8 @@ describe(`Configuration`, () => {
         const emptyEnvWithStrictFallback = getToken(`emptyEnvWithStrictFallback`);
         const emptyEnvWithFallback = getToken(`emptyEnvWithFallback`);
         const emptyEnvWithEmptyFallback = getToken(`emptyEnvWithEmptyFallback`);
+        const emptyEnvWithNestedEnv = getToken(`emptyEnvWithNestedEnv`);
+        const multipleNestedEmptyEnvWithFallback = getToken(`multipleNestedEmptyEnvWithFallback`);
 
         expect(onlyEnv).toEqual(`AAA-BBB-CCC`);
         expect(multipleEnvs).toEqual(`AAA-BBB-CCC-separator-AAA-BBB-CCC`);
@@ -142,6 +150,8 @@ describe(`Configuration`, () => {
         expect(emptyEnvWithStrictFallback).toEqual(``);
         expect(emptyEnvWithFallback).toEqual(`fallback-for-empty-value`);
         expect(emptyEnvWithEmptyFallback).toEqual(``);
+        expect(emptyEnvWithNestedEnv).toEqual(`AAA-BBB-CCC`);
+        expect(multipleNestedEmptyEnvWithFallback).toEqual(`fallback-value`);
       });
     });
 

packages/yarnpkg-core/tests/miscUtils.test.ts

@@ -18,13 +18,41 @@ describe(`miscUtils`, () => {
       ).toBe(`VAR_A: ValueA, VAR_B: ValueB`);
     });
 
+    it(`should use empty strings when environment variables are empty strings`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A-FallbackA}, VAR_B: \${VAR_B-FallbackB}`,
+          {
+            env: {
+              VAR_A: ``,
+              VAR_B: ``,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: , VAR_B: `);
+    });
+
     it(`should use fallback values when environment variables are not set`, () => {
       expect(
         miscUtils.replaceEnvVariables(
-          `VAR_A: \${VAR_A:-ValueA}, VAR_B: \${VAR_B:-ValueB}`,
+          `VAR_A: \${VAR_A:-FallbackA}, VAR_B: \${VAR_B:-FallbackB}`,
           {env: {}},
         ),
-      ).toBe(`VAR_A: ValueA, VAR_B: ValueB`);
+      ).toBe(`VAR_A: FallbackA, VAR_B: FallbackB`);
+    });
+
+    it(`should use fallback values when environment variables are empty strings`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-FallbackA}, VAR_B: \${VAR_B:-FallbackB}`,
+          {
+            env: {
+              VAR_A: ``,
+              VAR_B: ``,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: FallbackA, VAR_B: FallbackB`);
     });
 
     it(`should not replace escaped environment variables`, () => {
@@ -40,6 +68,122 @@ describe(`miscUtils`, () => {
         ),
       ).toBe(`VAR_A: \${VAR_A}, VAR_B: \${VAR_B}`);
     });
+
+    it(`should replace primary environment variables with their values when there is 1 step of nesting`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A-\${VAR_A2-FallbackA}}, VAR_B: \${VAR_B-\${VAR_B2-FallbackB}}`,
+          {
+            env: {
+              VAR_A: `ValueA`,
+              VAR_B: `ValueB`,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: ValueA, VAR_B: ValueB`);
+    });
+
+    it(`should use empty strings when primary environment variables are empty strings when there is 1 step of nesting`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A-\${VAR_A2-FallbackA}}, VAR_B: \${VAR_B-\${VAR_B2-FallbackB}}`,
+          {
+            env: {
+              VAR_A: ``,
+              VAR_B: ``,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: , VAR_B: `);
+    });
+
+    it(`should replace primary environment variables with their values when there is 1 step of nesting`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-\${VAR_A2:-FallbackA}}, VAR_B: \${VAR_B:-\${VAR_B2:-FallbackB}}`,
+          {
+            env: {
+              VAR_A: `ValueA`,
+              VAR_B: `ValueB`,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: ValueA, VAR_B: ValueB`);
+    });
+
+    it(`should replace secondary environment variables with their values when primary variables are not set`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-\${VAR_A2:-FallbackA}}, VAR_B: \${VAR_B:-\${VAR_B2:-FallbackB}}`,
+          {
+            env: {
+              VAR_A2: `ValueA2`,
+              VAR_B2: `ValueB2`,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: ValueA2, VAR_B: ValueB2`);
+    });
+
+    it(`should use fallback values when primary and secondary variables are not set`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-\${VAR_A2:-FallbackA}}, VAR_B: \${VAR_B:-\${VAR_B2:-FallbackB}}`,
+          {env: {}},
+        ),
+      ).toBe(`VAR_A: FallbackA, VAR_B: FallbackB`);
+    });
+
+    it(`should not replace escaped primary environment variables`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \\\${VAR_A:-\${VAR_A2:-FallbackA}}, VAR_B: \\\${VAR_B:-\${VAR_B2:-FallbackB}}`,
+          {
+            env: {
+              VAR_A: `ValueA`,
+              VAR_B: `ValueB`,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: \${VAR_A:-\${VAR_A2:-FallbackA}}, VAR_B: \${VAR_B:-\${VAR_B2:-FallbackB}}`);
+    });
+
+    it(`should not replace escaped secondary environment variables`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-\\\${VAR_A2:-FallbackA}}, VAR_B: \${VAR_B:-\\\${VAR_B2:-FallbackB}}`,
+          {
+            env: {
+              VAR_A2: `ValueA2`,
+              VAR_B2: `ValueB2`,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: \${VAR_A2:-FallbackA}, VAR_B: \${VAR_B2:-FallbackB}`);
+    });
+
+    it(`should replace tertiary environment variables with their values when primary and secondary variables are not set`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-\${VAR_A2:-\${VAR_A3:-FallbackA}}}, VAR_B: \${VAR_B:-\${VAR_B2:-\${VAR_B3:-FallbackB}}}`,
+          {
+            env: {
+              VAR_A3: `ValueA3`,
+              VAR_B3: `ValueB3`,
+            },
+          },
+        ),
+      ).toBe(`VAR_A: ValueA3, VAR_B: ValueB3`);
+    });
+
+    it(`should use fallback values when primary, secondary and tertiary variables are not set`, () => {
+      expect(
+        miscUtils.replaceEnvVariables(
+          `VAR_A: \${VAR_A:-\${VAR_A2:-\${VAR_A3:-FallbackA}}}, VAR_B: \${VAR_B:-\${VAR_B2:-\${VAR_B3:-FallbackB}}}`,
+          {env: {}},
+        ),
+      ).toBe(`VAR_A: FallbackA, VAR_B: FallbackB`);
+    });
   });
 
   describe(`mapAndFind`, () => {