diff --git a/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterReferenceUtils.java b/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterReferenceUtils.java new file mode 100644 index 000000000000..b11fa64bfda5 --- /dev/null +++ b/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterReferenceUtils.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter; + +import java.util.List; + +/** + * Utility methods related to parameter references in parameter values. + */ +public final class ParameterReferenceUtils { + + private static final ParameterParser PARSER = new ExpressionLanguageAgnosticParameterParser(); + + private ParameterReferenceUtils() { + } + + /** + * Returns the referenced parameter name if {@code value} is a one-to-one parameter reference, + * meaning its entire content is exactly a single parameter reference token such as + * {@code #{foo}} or {@code #{'My Parameter'}}. Returns {@code null} if {@code value} is not a + * one-to-one parameter reference (for example, if it is {@code null}, empty, contains literal + * text in addition to a reference, contains multiple references, contains an escaped + * reference, or is malformed). + * + * @param value the value to inspect + * @return the referenced parameter name, or {@code null} if {@code value} is not a one-to-one + * parameter reference + */ + public static String extractOneToOneParameterReference(final String value) { + if (value == null || value.isEmpty()) { + return null; + } + final List tokens = PARSER.parseTokens(value).toList(); + if (tokens.size() != 1) { + return null; + } + final ParameterToken token = tokens.getFirst(); + if (!token.isParameterReference()) { + return null; + } + if (token.getStartOffset() != 0 || token.getEndOffset() != value.length() - 1) { + return null; + } + return ((ParameterReference) token).getParameterName(); + } +} diff --git a/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestParameterReferenceUtils.java b/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestParameterReferenceUtils.java new file mode 100644 index 000000000000..92b16e5d7eca --- /dev/null +++ b/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestParameterReferenceUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class TestParameterReferenceUtils { + + @Test + public void testExtractOneToOneParameterReferenceMatches() { + assertEquals("foo", ParameterReferenceUtils.extractOneToOneParameterReference("#{foo}")); + assertEquals("a-b_c.d", ParameterReferenceUtils.extractOneToOneParameterReference("#{a-b_c.d}")); + assertEquals("My Parameter", ParameterReferenceUtils.extractOneToOneParameterReference("#{'My Parameter'}")); + assertEquals("with.dot", ParameterReferenceUtils.extractOneToOneParameterReference("#{'with.dot'}")); + } + + @Test + public void testExtractOneToOneParameterReferenceRejectsNonOneToOneValues() { + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference(null)); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("plain text")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("prefix #{foo}")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("#{foo}suffix")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("#{a}#{b}")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("##{foo}")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("#{abc")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("#{a}b}}")); + assertNull(ParameterReferenceUtils.extractOneToOneParameterReference("${#{foo}}")); + } +} diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc index 3d01553a4290..eca17460661d 100644 --- a/nifi-docs/src/main/asciidoc/user-guide.adoc +++ b/nifi-docs/src/main/asciidoc/user-guide.adoc @@ -1047,6 +1047,29 @@ Sensitive properties may only reference sensitive Parameters. This is important The value of a sensitive property must be set to a single Parameter reference. For example, values of `+#{password}123+` and `+#{password}#{suffix}+` are not allowed. Sending `+#{password}123+` would lead to exposing part of the sensitive property's value. This is in contrast to a non-sensitive property, where a value such as `+#{path}/child/file.txt+` is valid. +[[parameter-value-references]] +===== Parameter Value References + +A Parameter's *value* may itself be a reference to another Parameter. When a Parameter's value is exactly `+#{otherParameter}+` (or `+#{'Other Parameter'}+` for names containing characters that are otherwise disallowed in unquoted references, such as spaces), the referencing Parameter is treated as a one-to-one alias of the referenced Parameter: at resolution time the alias's effective value is replaced with the referenced Parameter's value. + +This is a one-to-one alias only. The Parameter's value must be a single reference and nothing else: literal text combined with a reference (`+jdbc://#{db_host}:3306+`) or multiple references (`+#{a}#{b}+`) are left as-is and not treated as aliases. + +The referenced Parameter may be any Parameter visible in the bound Parameter Context's effective scope: a Parameter defined in the same Context, a Parameter inherited from another Parameter Context, or a Parameter sourced from a <>. The referencing and referenced Parameters must have the same sensitivity; otherwise the reference is left unresolved. + +Resolution is performed a single level deep -- aliases of aliases are not followed. For example, if Parameter `A` has value `+#{B}+` and Parameter `B` has value `+#{C}+`, then `A` resolves to the literal value `+#{C}+`, not to the value of `C`. + +Updating the referenced Parameter's value cascades transparently: components configured with the alias are identified as affected components and are stopped, validated, and restarted as part of the Parameter Context update, just as if they had referenced the underlying Parameter directly. + +To illustrate, assume a Parameter Context `Shared` defines `db_host` with value `myserver.example.com`, and a Parameter Context `App` inherits from `Shared` and defines an alias `host` with value `+#{db_host}+`. A processor bound to `App` configured with `+#{host}+` resolves to `myserver.example.com`. + +|==== +| *Aliasing Parameter (in App)* | *Referenced Parameter (in Shared)* | *Effective Property Value* +| `host` = `+#{db_host}+` | `db_host` = `myserver.example.com` | `myserver.example.com` +| `password` = `+#{secret_value}+` (sensitive) | `secret_value` = `s3cr3t` (sensitive) | `s3cr3t` +| `url` = `+jdbc://#{db_host}:3306+` | `db_host` = `myserver.example.com` | `+jdbc://#{db_host}:3306+` (not an alias - literal text plus reference) +|==== + + ==== Parameter Providers Parameter Providers allow parameters to be stored in sources external to NiFi (e.g. HashiCorp Vault). The parameters of a Parameter Provider can be fetched and applied to all referencing Parameter Contexts. diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java index 755f48f93a70..3abd69197bf5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java @@ -81,6 +81,7 @@ import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterReference; +import org.apache.nifi.parameter.ParameterReferenceUtils; import org.apache.nifi.parameter.ParameterUpdate; import org.apache.nifi.parameter.StandardParameterUpdate; import org.apache.nifi.processor.DataUnit; @@ -3326,13 +3327,56 @@ public void setParameterContext(final ParameterContext parameterContext) { public void onParameterContextUpdated(final Map updatedParameters) { readLock.lock(); try { - getProcessors().forEach(proc -> proc.onParametersModified(updatedParameters)); - getControllerServices(false).forEach(cs -> cs.onParametersModified(updatedParameters)); + final Map effectiveUpdates = augmentWithParameterValueReferences(updatedParameters); + getProcessors().forEach(proc -> proc.onParametersModified(effectiveUpdates)); + getControllerServices(false).forEach(cs -> cs.onParametersModified(effectiveUpdates)); } finally { readLock.unlock(); } } + /** + * Augments the given parameter update map with entries for local parameters whose values are + * one-to-one references to changed parameters. For example, if this group's context defines + * parameter X with value {@code #{db_host}} and db_host is in the update map, then X is added + * to the augmented map with the same old/new values, allowing components referencing X to be + * properly notified of the change. The referenced parameter may be any parameter visible in + * the bound context's effective scope (local, inherited from a user-managed context, or + * sourced from a Parameter Provider). + */ + private Map augmentWithParameterValueReferences(final Map updatedParameters) { + final ParameterContext context = getParameterContext(); + if (context == null) { + return updatedParameters; + } + + Map augmented = null; + for (final Map.Entry entry : context.getParameters().entrySet()) { + final Parameter localParam = entry.getValue(); + final String referencedName = ParameterReferenceUtils.extractOneToOneParameterReference(localParam.getValue()); + if (referencedName == null) { + continue; + } + + final Optional referencedParam = context.getParameter(referencedName); + if (referencedParam.isEmpty()) { + continue; + } + + final ParameterUpdate referencedUpdate = updatedParameters.get(referencedName); + if (referencedUpdate != null && localParam.getDescriptor().isSensitive() == referencedUpdate.isSensitive()) { + if (augmented == null) { + augmented = new HashMap<>(updatedParameters); + } + augmented.put(localParam.getDescriptor().getName(), + new StandardParameterUpdate(localParam.getDescriptor().getName(), + referencedUpdate.getPreviousValue(), referencedUpdate.getUpdatedValue(), + localParam.getDescriptor().isSensitive())); + } + } + return augmented != null ? augmented : updatedParameters; + } + private Map mapParameterUpdates(final ParameterContext previousParameterContext, final ParameterContext updatedParameterContext) { if (previousParameterContext == null && updatedParameterContext == null) { return Collections.emptyMap(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java index 5bc98968961e..59655ecc5108 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java @@ -346,7 +346,9 @@ public Map getParameters() { public Map getEffectiveParameters() { readLock.lock(); try { - return this.getEffectiveParameters(inheritedParameterContexts); + final Map effective = getMergedEffectiveParameters(inheritedParameterContexts, this.parameters); + resolveParameterValueReferences(effective); + return effective; } finally { readLock.unlock(); } @@ -358,7 +360,8 @@ public Map getEffectiveParameterUpdates(final Map currentEffectiveParameters = getEffectiveParameters(); - final Map effectiveProposedParameters = getEffectiveParameters(inheritedParameterContexts, getProposedParameters(parameterUpdates)); + final Map effectiveProposedParameters = getMergedEffectiveParameters(inheritedParameterContexts, getProposedParameters(parameterUpdates)); + resolveParameterValueReferences(effectiveProposedParameters); return getEffectiveParameterUpdates(currentEffectiveParameters, effectiveProposedParameters); } @@ -370,7 +373,9 @@ public Map getEffectiveParameterUpdates(final Map getEffectiveParameters(final Map proposedParameters) { - return getEffectiveParameters(this.inheritedParameterContexts, proposedParameters); + final Map effective = getMergedEffectiveParameters(this.inheritedParameterContexts, proposedParameters); + resolveParameterValueReferences(effective); + return effective; } /** @@ -380,31 +385,100 @@ private Map getEffectiveParameters(final Map getEffectiveParameters(final List parameterContexts) { - return getEffectiveParameters(parameterContexts, this.parameters); + final Map effective = getMergedEffectiveParameters(parameterContexts, this.parameters); + resolveParameterValueReferences(effective); + return effective; } - private Map getEffectiveParameters(final List parameterContexts, - final Map proposedParameters) { - return getEffectiveParameters(parameterContexts, proposedParameters, new HashMap<>()); + private Map getMergedEffectiveParameters(final List parameterContexts, + final Map proposedParameters) { + return getMergedEffectiveParameters(parameterContexts, proposedParameters, new HashMap<>()); } - private Map getEffectiveParameters(final List parameterContexts, - final Map proposedParameters, - final Map> allOverrides) { + /** + * Merges parameters from inherited contexts with the proposed (local) parameters, applying + * override priority. Does NOT resolve parameter value references -- callers that need resolved + * values must call {@link #resolveParameterValueReferences} on the result. + */ + private Map getMergedEffectiveParameters(final List parameterContexts, + final Map proposedParameters, + final Map> allOverrides) { final Map effectiveParameters = new LinkedHashMap<>(); - // Loop backwards so that the first ParameterContext in the list will override any parameters later in the list for (int i = parameterContexts.size() - 1; i >= 0; i--) { - ParameterContext parameterContext = parameterContexts.get(i); - combineOverrides(allOverrides, overrideParameters(effectiveParameters, parameterContext.getEffectiveParameters(), parameterContext)); + final ParameterContext parameterContext = parameterContexts.get(i); + final Map inheritedParameters = getUnresolvedEffectiveParameters(parameterContext); + combineOverrides(allOverrides, overrideParameters(effectiveParameters, inheritedParameters, parameterContext)); } - // Finally, override all child parameters with our own combineOverrides(allOverrides, overrideParameters(effectiveParameters, proposedParameters, this)); return effectiveParameters; } + /** + * Returns the merged effective parameters from a context without applying parameter value + * reference resolution. For StandardParameterContext instances this avoids double-resolution + * when building a parent context's effective parameter set. + */ + private static Map getUnresolvedEffectiveParameters(final ParameterContext parameterContext) { + if (parameterContext instanceof StandardParameterContext standardContext) { + return standardContext.getMergedEffectiveParametersReadLocked(); + } + return parameterContext.getEffectiveParameters(); + } + + private Map getMergedEffectiveParametersReadLocked() { + readLock.lock(); + try { + return getMergedEffectiveParameters(inheritedParameterContexts, this.parameters); + } finally { + readLock.unlock(); + } + } + + /** + * Resolves one-to-one parameter value references within the effective parameter map. + * If a parameter's entire value is exactly {@code #{referencedName}}, and the referenced parameter + * exists in the effective map with matching sensitivity, the value is replaced with the referenced + * parameter's value. The referenced parameter may be any parameter visible in the merged effective + * scope -- it may come from the same context, from an inherited context, or be sourced from a + * Parameter Provider. Only a single level of resolution is performed (no chaining): the lookup uses + * a snapshot of the pre-resolution values so that transitive references are not followed. + * + * @param effectiveParameters the effective parameter map to resolve in place + */ + private void resolveParameterValueReferences(final Map effectiveParameters) { + final Map originalParametersByName = new HashMap<>(); + for (final Map.Entry entry : effectiveParameters.entrySet()) { + originalParametersByName.put(entry.getKey().getName(), entry.getValue()); + } + + for (final Map.Entry entry : effectiveParameters.entrySet()) { + final ParameterDescriptor descriptor = entry.getKey(); + final Parameter parameter = entry.getValue(); + final String referencedName = ParameterReferenceUtils.extractOneToOneParameterReference(parameter.getValue()); + if (referencedName == null) { + continue; + } + + final Parameter referencedParameter = originalParametersByName.get(referencedName); + if (referencedParameter == null) { + continue; + } + + if (descriptor.isSensitive() != referencedParameter.getDescriptor().isSensitive()) { + continue; + } + + final Parameter resolvedParameter = new Parameter.Builder() + .fromParameter(parameter) + .value(referencedParameter.getValue()) + .build(); + entry.setValue(resolvedParameter); + } + } + private void combineOverrides(final Map> existingOverrides, final Map> newOverrides) { for (final Map.Entry> entry : newOverrides.entrySet()) { final ParameterDescriptor key = entry.getKey(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java index d4641821e4b3..e350056e64e3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java @@ -903,12 +903,25 @@ private static ParameterDescriptor addParameter(final ParameterContext parameter } private static ParameterDescriptor addParameter(final ParameterContext parameterContext, final String name, final String value, final boolean isSensitive) { + return addParameter(parameterContext, name, value, isSensitive, false); + } + + private static ParameterDescriptor addProvidedParameter(final ParameterContext parameterContext, final String name, final String value) { + return addParameter(parameterContext, name, value, false, true); + } + + private static ParameterDescriptor addProvidedParameter(final ParameterContext parameterContext, final String name, final String value, final boolean isSensitive) { + return addParameter(parameterContext, name, value, isSensitive, true); + } + + private static ParameterDescriptor addParameter(final ParameterContext parameterContext, final String name, final String value, + final boolean isSensitive, final boolean isProvided) { final Map parameters = new HashMap<>(); for (final Map.Entry entry : parameterContext.getParameters().entrySet()) { parameters.put(entry.getKey().getName(), entry.getValue()); } final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(name).sensitive(isSensitive).build(); - parameters.put(name, createParameter(parameterDescriptor, value)); + parameters.put(name, createParameter(parameterDescriptor, value, isProvided)); parameterContext.setParameters(parameters); return parameterDescriptor; } @@ -927,6 +940,335 @@ private static ParameterContext createParameterContext(final String id, final Pa return parameterContext; } + @Test + public void testParameterValueReferenceResolution() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "db_host", "myserver.example.com"); + addProvidedParameter(s, "db_port", "3306"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "host", "#{db_host}"); + addParameter(p, "port", "#{db_port}"); + addParameter(p, "plain", "literal_value"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("myserver.example.com", effective.get(new ParameterDescriptor.Builder().name("host").build()).getValue()); + assertEquals("3306", effective.get(new ParameterDescriptor.Builder().name("port").build()).getValue()); + assertEquals("literal_value", effective.get(new ParameterDescriptor.Builder().name("plain").build()).getValue()); + assertEquals("myserver.example.com", effective.get(new ParameterDescriptor.Builder().name("db_host").build()).getValue()); + } + + @Test + public void testParameterValueReferenceResolvesQuotedName() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addParameter(s, "My Parameter", "expected_value"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "alias", "#{'My Parameter'}"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("expected_value", effective.get(new ParameterDescriptor.Builder().name("alias").build()).getValue()); + } + + @Test + public void testParameterValueReferenceNotResolvedIfMixed() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "db_host", "myserver.example.com"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "url", "jdbc://#{db_host}:3306"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("jdbc://#{db_host}:3306", effective.get(new ParameterDescriptor.Builder().name("url").build()).getValue()); + } + + @Test + public void testParameterValueReferenceNotResolvedIfMultipleRefs() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "a", "valueA"); + addProvidedParameter(s, "b", "valueB"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "combined", "#{a}#{b}"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("#{a}#{b}", effective.get(new ParameterDescriptor.Builder().name("combined").build()).getValue()); + } + + @Test + public void testParameterValueReferenceNoChaining() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "y", "#{z}"); + addProvidedParameter(s, "z", "final_value"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "x", "#{y}"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + // x references y, whose value is "#{z}" -- no chaining, so x resolves to "#{z}" literally + assertEquals("#{z}", effective.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + } + + @Test + public void testParameterValueReferenceSensitivityMatchResolves() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "secret_value", "my_secret", true); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "password", "#{secret_value}", true); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("my_secret", effective.get(new ParameterDescriptor.Builder().name("password").sensitive(true).build()).getValue()); + } + + @Test + public void testParameterValueReferenceSensitivityMismatchNotResolved() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "secret_value", "my_secret", true); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "not_sensitive", "#{secret_value}"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("#{secret_value}", effective.get(new ParameterDescriptor.Builder().name("not_sensitive").build()).getValue()); + } + + @Test + public void testParameterValueReferenceSensitivityMismatchSensitiveRefNonSensitive() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "plain_value", "not_a_secret"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "sensitive_param", "#{plain_value}", true); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("#{plain_value}", effective.get(new ParameterDescriptor.Builder().name("sensitive_param").sensitive(true).build()).getValue()); + } + + @Test + public void testParameterValueReferenceNonExistent() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "ref", "#{nonexistent}"); + + final Map effective = p.getEffectiveParameters(); + assertEquals("#{nonexistent}", effective.get(new ParameterDescriptor.Builder().name("ref").build()).getValue()); + } + + @Test + public void testParameterValueReferenceSelfReference() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "x", "#{x}"); + + final Map effective = p.getEffectiveParameters(); + // Self-reference: x's value is "#{x}". One level of resolution replaces x's value with the + // referenced parameter's value, which is still "#{x}", so it stays as "#{x}". + assertEquals("#{x}", effective.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + } + + @Test + public void testGetParametersReturnsRawValues() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addProvidedParameter(s, "db_host", "myserver.example.com"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "host", "#{db_host}"); + + p.setInheritedParameterContexts(List.of(s)); + + // getParameters() returns only local parameters with raw (unresolved) values + final Map raw = p.getParameters(); + assertEquals(1, raw.size()); + assertEquals("#{db_host}", raw.get(new ParameterDescriptor.Builder().name("host").build()).getValue()); + } + + @Test + public void testParameterValueReferenceSameContextProvidedResolution() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addProvidedParameter(p, "source", "resolved_value"); + addParameter(p, "ref", "#{source}"); + + final Map effective = p.getEffectiveParameters(); + assertEquals("resolved_value", effective.get(new ParameterDescriptor.Builder().name("ref").build()).getValue()); + } + + @Test + public void testParameterValueReferenceResolvedFromInheritedUserParameter() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext s = createParameterContext("s", parameterContextLookup); + addParameter(s, "db_host", "myserver.example.com"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "host", "#{db_host}"); + + p.setInheritedParameterContexts(List.of(s)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("myserver.example.com", effective.get(new ParameterDescriptor.Builder().name("host").build()).getValue()); + } + + @Test + public void testParameterValueReferenceSameContextUserParameterResolved() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "source", "some_value"); + addParameter(p, "ref", "#{source}"); + + final Map effective = p.getEffectiveParameters(); + assertEquals("some_value", effective.get(new ParameterDescriptor.Builder().name("ref").build()).getValue()); + } + + @Test + public void testParameterValueReferenceResolvedAcrossMultiLevelInheritance() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "target", "deep_value"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + b.setInheritedParameterContexts(List.of(c)); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "alias", "#{target}"); + a.setInheritedParameterContexts(List.of(b)); + + final Map effective = a.getEffectiveParameters(); + assertEquals("deep_value", effective.get(new ParameterDescriptor.Builder().name("alias").build()).getValue()); + } + + @Test + public void testParameterValueReferenceFromLowerPrioritySiblingToHigherPriority() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "paramInA", "valueA"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "x", "#{paramInA}"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + p.setInheritedParameterContexts(List.of(a, b, c)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("valueA", effective.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + assertEquals("valueA", effective.get(new ParameterDescriptor.Builder().name("paramInA").build()).getValue()); + } + + @Test + public void testParameterValueReferenceFromLowerPriorityIsHiddenWhenOverridden() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "paramInA", "valueA"); + addParameter(a, "x", "fromA"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "x", "#{paramInA}"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + p.setInheritedParameterContexts(List.of(a, b, c)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("fromA", effective.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + } + + @Test + public void testParameterValueReferenceFromLowerPriorityResolvesToHigherPriorityWhenNameOverlaps() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "paramInA", "valueFromA"); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "paramInA", "valueFromC"); + addParameter(c, "x", "#{paramInA}"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + p.setInheritedParameterContexts(List.of(a, c)); + + final Map effective = p.getEffectiveParameters(); + assertEquals("valueFromA", effective.get(new ParameterDescriptor.Builder().name("paramInA").build()).getValue()); + assertEquals("valueFromA", effective.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + } + + @Test + public void testParameterValueReferenceNotResolvedWhenQueriedDirectlyOnChild() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "paramInA", "valueA"); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "x", "#{paramInA}"); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + p.setInheritedParameterContexts(List.of(a, c)); + + final Map effectiveP = p.getEffectiveParameters(); + assertEquals("valueA", effectiveP.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + + final Map effectiveC = c.getEffectiveParameters(); + assertEquals("#{paramInA}", effectiveC.get(new ParameterDescriptor.Builder().name("x").build()).getValue()); + } + + @Test + public void testParameterValueReferenceMutualReferencesStaySingleLevel() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext p = createParameterContext("p", parameterContextLookup); + addParameter(p, "a", "#{b}"); + addParameter(p, "b", "#{a}"); + + final Map effective = p.getEffectiveParameters(); + assertEquals("#{a}", effective.get(new ParameterDescriptor.Builder().name("a").build()).getValue()); + assertEquals("#{b}", effective.get(new ParameterDescriptor.Builder().name("b").build()).getValue()); + } + private static class HashMapParameterReferenceManager implements ParameterReferenceManager { private final Map processors = new HashMap<>(); private final Map controllerServices = new HashMap<>(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index c6fcb7fb93e0..dc82f31094a6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -173,6 +173,7 @@ import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.parameter.ParameterReferenceManager; +import org.apache.nifi.parameter.ParameterReferenceUtils; import org.apache.nifi.parameter.StandardParameterContext.Builder; import org.apache.nifi.processor.Processor; import org.apache.nifi.processor.VerifiableProcessor; @@ -1681,6 +1682,12 @@ private Set getComponentsAffectedByParameterContextUpda final Set updatedParameterNames = getUpdatedParameterNames(parameterContextDto); + // Extend the updated parameter names with cascading names from parameter value references. + // If a process group is bound to a context P whose local parameter X has the value #{Y}, and Y is + // any parameter visible in P's effective scope (local, inherited, or provider-sourced) that appears + // in the update set, then X is effectively updated as well. + final Set extendedParameterNames = extendWithParameterValueReferences(updatedParameterNames, groupsReferencingParameterContext); + // Clear set of Affected Components for each Parameter. This parameter is read-only and it will be populated below. for (final ParameterEntity parameterEntity : parameterContextDto.getParameters()) { parameterEntity.getParameter().setReferencingComponents(new HashSet<>()); @@ -1698,10 +1705,12 @@ private Set getComponentsAffectedByParameterContextUpda } } + final ParameterContext groupContext = group.getParameterContext(); + for (final ProcessorNode processor : group.getProcessors()) { if (includeInactive || processor.isRunning()) { final Set referencedParams = processor.getReferencedParameterNames(); - final boolean referencesUpdatedParam = referencedParams.stream().anyMatch(updatedParameterNames::contains); + final boolean referencesUpdatedParam = referencedParams.stream().anyMatch(extendedParameterNames::contains); if (referencesUpdatedParam) { affectedComponents.add(processor); @@ -1711,7 +1720,7 @@ private Set getComponentsAffectedByParameterContextUpda for (final String referencedParam : referencedParams) { for (final ParameterEntity paramEntity : parameterContextDto.getParameters()) { final ParameterDTO paramDto = paramEntity.getParameter(); - if (referencedParam.equals(paramDto.getName())) { + if (referencesContextParameter(referencedParam, paramDto.getName(), groupContext)) { paramDto.getReferencingComponents().add(affectedComponentEntity); } } @@ -1723,13 +1732,13 @@ private Set getComponentsAffectedByParameterContextUpda for (final ControllerServiceNode service : group.getControllerServices(false)) { if (includeInactive || service.isActive()) { final Set referencedParams = service.getReferencedParameterNames(); - final Set updatedReferencedParams = referencedParams.stream().filter(updatedParameterNames::contains).collect(Collectors.toSet()); + final Set updatedReferencedParams = referencedParams.stream().filter(extendedParameterNames::contains).collect(Collectors.toSet()); final List affectedParameterDtos = new ArrayList<>(); for (final String referencedParam : referencedParams) { for (final ParameterEntity paramEntity : parameterContextDto.getParameters()) { final ParameterDTO paramDto = paramEntity.getParameter(); - if (referencedParam.equals(paramDto.getName())) { + if (referencesContextParameter(referencedParam, paramDto.getName(), groupContext)) { affectedParameterDtos.add(paramDto); } } @@ -1857,6 +1866,52 @@ private Set getUpdatedParameterNames(final ParameterContextDTO parameter return updatedParameters; } + private Set extendWithParameterValueReferences(final Set updatedParameterNames, final List groupsReferencingParameterContext) { + final Set extended = new HashSet<>(updatedParameterNames); + for (final ProcessGroup group : groupsReferencingParameterContext) { + final ParameterContext groupContext = group.getParameterContext(); + if (groupContext == null) { + continue; + } + for (final Map.Entry entry : groupContext.getParameters().entrySet()) { + final String referencedName = ParameterReferenceUtils.extractOneToOneParameterReference(entry.getValue().getValue()); + if (referencedName == null || !updatedParameterNames.contains(referencedName)) { + continue; + } + final Optional referencedParam = groupContext.getParameter(referencedName); + if (referencedParam.isPresent()) { + extended.add(entry.getKey().getName()); + } + } + } + return extended; + } + + /** + * Returns true if a component property's reference to {@code referencedParamName} is logically equivalent to + * referencing {@code contextParamName} in the updated context. This is true either when the names match directly + * or when {@code groupContext} contains a local parameter named {@code referencedParamName} whose value is the + * one-to-one parameter reference {@code #{contextParamName}} (i.e. an alias). The alias case lets the affected + * component be linked to the source parameter being updated, even when the property text uses the alias name. + */ + private boolean referencesContextParameter(final String referencedParamName, final String contextParamName, final ParameterContext groupContext) { + if (referencedParamName.equals(contextParamName)) { + return true; + } + if (groupContext == null) { + return false; + } + final Parameter localParam = groupContext.getParameters().get(new ParameterDescriptor.Builder().name(referencedParamName).build()); + if (localParam == null) { + return false; + } + final String aliasTarget = ParameterReferenceUtils.extractOneToOneParameterReference(localParam.getValue()); + if (!contextParamName.equals(aliasTarget)) { + return false; + } + return groupContext.getParameter(contextParamName).isPresent(); + } + @Override public ProcessGroupEntity updateProcessGroup(final Revision revision, final ProcessGroupDTO processGroupDTO) { final ProcessGroup processGroupNode = processGroupDAO.getProcessGroup(processGroupDTO.getId()); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java index 40a609bd79b5..987dc7cc00bc 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java @@ -1295,6 +1295,117 @@ protected void assertAssetExists(final AssetEntity asset, final AssetsEntity ass assertNotNull(assetFromListing); } + @Test + public void testParameterValueReferenceUpdatePropagation() throws NiFiClientException, IOException, InterruptedException { + // Create a parameter provider for context S with a parameter db_host + ParameterProviderEntity parameterProvider = createParameterProvider("PropertiesParameterProvider"); + parameterProvider = updateParameterProviderProperties(parameterProvider, Collections.singletonMap("parameters", "db_host=0 sec")); + + final String parameterGroupName = "Parameters"; + final String sContextName = "S_Context"; + final ParameterContextEntity sContextEntity = createParameterContextEntity(sContextName, "Inherited provider context", + Collections.emptySet(), Collections.emptyList(), parameterProvider, parameterGroupName); + final ParameterContextEntity createdS = getNifiClient().getParamContextClient().createParamContext(sContextEntity); + + // Fetch and apply parameters from the provider so db_host becomes a provided parameter + final ParameterGroupConfigurationEntity groupConfiguration = new ParameterGroupConfigurationEntity(); + groupConfiguration.setSynchronized(true); + groupConfiguration.setGroupName(parameterGroupName); + groupConfiguration.setParameterContextName(sContextName); + groupConfiguration.setParameterSensitivities(Collections.singletonMap("db_host", ParameterSensitivity.NON_SENSITIVE)); + fetchAndWaitForAppliedParameters(parameterProvider, Collections.singletonList(groupConfiguration)); + + // Create parent context P with parameter host = #{db_host}, inheriting from S + final Set pParams = new HashSet<>(); + pParams.add(createParameterEntity("host", null, false, "#{db_host}")); + final ParameterContextEntity pContextEntity = createParameterContextEntity("P_Context", "Parent context", + pParams, Collections.singletonList(createdS), null, null); + final ParameterContextEntity createdP = getNifiClient().getParamContextClient().createParamContext(pContextEntity); + + // Bind the root process group to P + setParameterContext("root", createdP); + + // Create a processor that references #{host} + ProcessorEntity processorEntity = createProcessor(TEST_PROCESSORS_PACKAGE + ".Sleep", NIFI_GROUP_ID, TEST_EXTENSIONS_ARTIFACT_ID, getNiFiVersion()); + final String processorId = processorEntity.getId(); + + final ProcessorConfigDTO config = processorEntity.getComponent().getConfig(); + config.setProperties(Collections.singletonMap("Validate Sleep Time", "#{host}")); + config.setAutoTerminatedRelationships(Collections.singleton("success")); + getNifiClient().getProcessorClient().updateProcessor(processorEntity); + + // host resolves to "0 sec" (from the provider), so the processor should be valid + waitForValidProcessor(processorId); + + // Start the processor + getClientUtil().startProcessor(processorEntity); + waitForRunningProcessor(processorId); + + try { + // Update db_host via the provider to a new value + parameterProvider = updateParameterProviderProperties(parameterProvider, Collections.singletonMap("parameters", "db_host=1 sec")); + fetchAndWaitForAppliedParameters(parameterProvider, Collections.singletonList(groupConfiguration)); + + // Processor should be running again after the parameter update completes + waitForRunningProcessor(processorId); + } finally { + getClientUtil().stopProcessor(processorEntity); + getNifiClient().getProcessorClient().deleteProcessor(processorId, processorEntity.getRevision().getClientId(), 3); + } + } + + @Test + public void testParameterValueReferenceUserManagedUpdatePropagation() throws NiFiClientException, IOException, InterruptedException { + // Create user-managed inherited context S with a parameter db_host (no parameter provider involved) + final Set sParams = new HashSet<>(); + sParams.add(createParameterEntity("db_host", null, false, "0 sec")); + final ParameterContextEntity sContextEntity = createParameterContextEntity("S_UserContext", "User-managed inherited context", + sParams, Collections.emptyList(), null, null); + final ParameterContextEntity createdS = getNifiClient().getParamContextClient().createParamContext(sContextEntity); + + // Create parent context P with parameter host = #{db_host}, inheriting from S + final Set pParams = new HashSet<>(); + pParams.add(createParameterEntity("host", null, false, "#{db_host}")); + final ParameterContextEntity pContextEntity = createParameterContextEntity("P_UserContext", "Parent context with alias", + pParams, Collections.singletonList(createdS), null, null); + final ParameterContextEntity createdP = getNifiClient().getParamContextClient().createParamContext(pContextEntity); + + setParameterContext("root", createdP); + + ProcessorEntity processorEntity = createProcessor(TEST_PROCESSORS_PACKAGE + ".Sleep", NIFI_GROUP_ID, TEST_EXTENSIONS_ARTIFACT_ID, getNiFiVersion()); + final String processorId = processorEntity.getId(); + + final ProcessorConfigDTO config = processorEntity.getComponent().getConfig(); + config.setProperties(Collections.singletonMap("Validate Sleep Time", "#{host}")); + config.setAutoTerminatedRelationships(Collections.singleton("success")); + getNifiClient().getProcessorClient().updateProcessor(processorEntity); + + // host resolves to "0 sec" via the alias to db_host in S, so the processor should validate + waitForValidProcessor(processorId); + + getClientUtil().startProcessor(processorEntity); + waitForRunningProcessor(processorId); + + try { + // Update db_host on S via a regular parameter context PUT (no provider) + final ParameterContextUpdateRequestEntity updateRequestEntity = updateParameterContext(createdS, "db_host", "1 sec"); + + // The processor references #{host}, which aliases #{db_host}; it must appear as an affected component + final Set affectedComponents = updateRequestEntity.getRequest().getReferencingComponents(); + final Set affectedComponentIds = affectedComponents.stream() + .map(AffectedComponentEntity::getId) + .collect(Collectors.toSet()); + assertTrue(affectedComponentIds.contains(processorId)); + + getClientUtil().waitForParameterContextRequestToComplete(createdS.getId(), updateRequestEntity.getRequest().getRequestId()); + + waitForRunningProcessor(processorId); + } finally { + getClientUtil().stopProcessor(processorEntity); + getNifiClient().getProcessorClient().deleteProcessor(processorId, processorEntity.getRevision().getClientId(), 3); + } + } + protected void assertAsset(final AssetEntity asset, final String expectedName) { assertNotNull(asset); assertNotNull(asset.getAsset());