[WIP] - Upstream 16530 - Optimize HostList API: conditional DISTINCT + composite index on JobHostSummary

Makefile

@@ -46,7 +46,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
-VENV_BOOTSTRAP ?= pip==26.1.2 setuptools==80.9.0 setuptools_scm[toml]==9.2.2 wheel==0.46.2
+VENV_BOOTSTRAP ?= pip==26.1.2 setuptools==83.0.0 setuptools_scm[toml]==9.2.2 wheel==0.46.2
 
 NAME ?= awx
 

awx/api/serializers.py

@@ -1499,6 +1499,13 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
     status = serializers.ChoiceField(choices=Project.PROJECT_STATUS_CHOICES, read_only=True)
     last_update_failed = serializers.BooleanField(read_only=True)
     last_updated = serializers.DateTimeField(read_only=True)
+    webhook_key = serializers.CharField(
+        write_only=True,
+        required=False,
+        allow_blank=True,
+        max_length=64,
+        help_text=_('Shared secret that the webhook service will use to sign requests. Leave blank to generate a new one when the webhook service is set.'),
+    )
     show_capabilities = ['start', 'schedule', 'edit', 'delete', 'copy']
     capabilities_prefetch = ['admin', 'update', {'copy': 'organization.project_admin'}]
 
@@ -1513,6 +1520,9 @@ class Meta:
             'allow_override',
             'default_environment',
             'signature_validation_credential',
+            'webhook_service',
+            'webhook_key',
+            'webhook_ref_filter',
         ) + (
             'last_update_failed',
             'last_updated',
@@ -1537,6 +1547,12 @@ def get_related(self, obj):
                 access_list=self.reverse('api:project_access_list', kwargs={'pk': obj.pk}),
                 object_roles=self.reverse('api:project_object_roles_list', kwargs={'pk': obj.pk}),
                 copy=self.reverse('api:project_copy', kwargs={'pk': obj.pk}),
+                webhook_key=self.reverse('api:webhook_key', kwargs={'model_kwarg': 'projects', 'pk': obj.pk}),
+                webhook_receiver=(
+                    self.reverse('api:webhook_receiver_{}'.format(obj.webhook_service), kwargs={'model_kwarg': 'projects', 'pk': obj.pk})
+                    if obj.webhook_service
+                    else ''
+                ),
             )
         )
         if obj.organization:
@@ -1581,6 +1597,10 @@ def get_field_from_model_or_attrs(fd):
             for fd in ('scm_update_on_launch', 'scm_delete_on_update', 'scm_track_submodules', 'scm_clean'):
                 if get_field_from_model_or_attrs(fd):
                     raise serializers.ValidationError({fd: _('Update options must be set to false for manual projects.')})
+        if get_field_from_model_or_attrs('webhook_service') and not get_field_from_model_or_attrs('scm_type'):
+            raise serializers.ValidationError({'webhook_service': _('Webhooks are not supported for manual projects.')})
+        if attrs.get('webhook_key') and not get_field_from_model_or_attrs('webhook_service'):
+            raise serializers.ValidationError({'webhook_key': _("Cannot set a webhook key without a webhook service.")})
         return super(ProjectSerializer, self).validate(attrs)
 
 
@@ -1625,7 +1645,7 @@ class Meta:
 class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer):
     class Meta:
         model = ProjectUpdate
-        fields = ('*', 'project', 'job_type', 'job_tags', '-controller_node')
+        fields = ('*', 'project', 'job_type', 'job_tags', '-controller_node', 'webhook_service', 'webhook_guid')
 
     def get_related(self, obj):
         res = super(ProjectUpdateSerializer, self).get_related(obj)
@@ -3180,6 +3200,7 @@ class Meta:
             'scm_branch',
             'forks',
             'limit',
+            'job_slice_pinned_hosts',
             'verbosity',
             'extra_vars',
             'job_tags',
@@ -3285,6 +3306,9 @@ def validate(self, attrs):
         webhook_service = attrs.get('webhook_service', getattr(self.instance, 'webhook_service', None))
         webhook_credential = attrs.get('webhook_credential', getattr(self.instance, 'webhook_credential', None))
 
+        if attrs.get('webhook_key') and not webhook_service:
+            raise serializers.ValidationError({'webhook_key': _("Cannot set a webhook key without a webhook service.")})
+
         if webhook_credential:
             if webhook_credential.credential_type.kind != 'token':
                 raise serializers.ValidationError({'webhook_credential': _("Must be a Personal Access Token.")})
@@ -3304,6 +3328,13 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
     capabilities_prefetch = ['admin', 'execute', {'copy': ['project.use', 'inventory.use']}]
 
     status = serializers.ChoiceField(choices=JobTemplate.JOB_TEMPLATE_STATUS_CHOICES, read_only=True, required=False)
+    webhook_key = serializers.CharField(
+        write_only=True,
+        required=False,
+        allow_blank=True,
+        max_length=64,
+        help_text=_('Shared secret that the webhook service will use to sign requests. Leave blank to generate a new one when the webhook service is set.'),
+    )
 
     class Meta:
         model = JobTemplate
@@ -3333,6 +3364,7 @@ class Meta:
             'job_slice_count',
             'webhook_service',
             'webhook_credential',
+            'webhook_key',
             'prevent_instance_group_fallback',
         )
         read_only_fields = ('*',)
@@ -3787,6 +3819,13 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
 
     skip_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
     job_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+    webhook_key = serializers.CharField(
+        write_only=True,
+        required=False,
+        allow_blank=True,
+        max_length=64,
+        help_text=_('Shared secret that the webhook service will use to sign requests. Leave blank to generate a new one when the webhook service is set.'),
+    )
 
     class Meta:
         model = WorkflowJobTemplate
@@ -3805,6 +3844,7 @@ class Meta:
             'ask_limit_on_launch',
             'webhook_service',
             'webhook_credential',
+            'webhook_key',
             '-execution_environment',
             'ask_labels_on_launch',
             'ask_skip_tags_on_launch',
@@ -3956,7 +3996,7 @@ class WorkflowApprovalSerializer(UnifiedJobSerializer):
 
     class Meta:
         model = WorkflowApproval
-        fields = ('*', '-controller_node', '-execution_node', 'can_approve_or_deny', 'approval_expiration', 'timed_out')
+        fields = ('*', '-controller_node', '-execution_node', 'can_approve_or_deny', 'approval_expiration', 'timed_out', 'context_message')
 
     def get_approval_expiration(self, obj):
         if obj.status != 'pending' or obj.timeout == 0:
@@ -3993,13 +4033,19 @@ class WorkflowApprovalActivityStreamSerializer(WorkflowApprovalSerializer):
 
 class WorkflowApprovalListSerializer(WorkflowApprovalSerializer, UnifiedJobListSerializer):
     class Meta:
-        fields = ('*', '-controller_node', '-execution_node', 'can_approve_or_deny', 'approval_expiration', 'timed_out')
+        fields = ('*', '-controller_node', '-execution_node', 'can_approve_or_deny', 'approval_expiration', 'timed_out', '-context_message')
+
+    def get_field_names(self, declared_fields, info):
+        field_names = super(WorkflowApprovalListSerializer, self).get_field_names(declared_fields, info)
+        # keep the potentially large rendered context out of list payloads,
+        # the detail view is the one that shows it
+        return tuple(x for x in field_names if x != 'context_message')
 
 
 class WorkflowApprovalTemplateSerializer(UnifiedJobTemplateSerializer):
     class Meta:
         model = WorkflowApprovalTemplate
-        fields = ('*', 'timeout', 'name')
+        fields = ('*', 'timeout', 'name', 'context_template')
 
     def get_related(self, obj):
         res = super(WorkflowApprovalTemplateSerializer, self).get_related(obj)
@@ -4159,10 +4205,27 @@ def validate(self, attrs):
         return attrs
 
 
+def _workflow_node_condition_edges(obj):
+    return [
+        {
+            'id': link.to_node_id,
+            'trigger': link.trigger,
+            'artifact_key': link.artifact_key,
+            'operator': link.operator,
+            'expected_value': link.expected_value,
+        }
+        for link in obj.condition_links_from.all()
+    ]
+
+
 class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
     success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
+    condition_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
+    condition_edges = serializers.SerializerMethodField(
+        help_text=_('List of condition data for each conditional edge originating from this node; id is the target node.')
+    )
     exclude_errors = ('required',)  # required variables may be provided by WFJT or on launch
 
     class Meta:
@@ -4179,16 +4242,23 @@ class Meta:
             'success_nodes',
             'failure_nodes',
             'always_nodes',
+            'condition_nodes',
+            'condition_edges',
             'all_parents_must_converge',
+            'max_retries',
             'identifier',
         )
 
+    def get_condition_edges(self, obj):
+        return _workflow_node_condition_edges(obj)
+
     def get_related(self, obj):
         res = super(WorkflowJobTemplateNodeSerializer, self).get_related(obj)
         res['create_approval_template'] = self.reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': obj.pk})
         res['success_nodes'] = self.reverse('api:workflow_job_template_node_success_nodes_list', kwargs={'pk': obj.pk})
         res['failure_nodes'] = self.reverse('api:workflow_job_template_node_failure_nodes_list', kwargs={'pk': obj.pk})
         res['always_nodes'] = self.reverse('api:workflow_job_template_node_always_nodes_list', kwargs={'pk': obj.pk})
+        res['condition_nodes'] = self.reverse('api:workflow_job_template_node_condition_nodes_list', kwargs={'pk': obj.pk})
         if obj.unified_job_template:
             res['unified_job_template'] = obj.unified_job_template.get_absolute_url(self.context.get('request'))
         try:
@@ -4216,6 +4286,11 @@ class WorkflowJobNodeSerializer(LaunchConfigurationBaseSerializer):
     success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
+    condition_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
+    condition_edges = serializers.SerializerMethodField(
+        help_text=_('List of condition data for each conditional edge originating from this node; id is the target node.')
+    )
+    retried_jobs = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
 
     class Meta:
         model = WorkflowJobNode
@@ -4232,18 +4307,27 @@ class Meta:
             'success_nodes',
             'failure_nodes',
             'always_nodes',
+            'condition_nodes',
+            'condition_edges',
             'all_parents_must_converge',
             'do_not_run',
             'prior_run_succeeded',
             'prior_run_elapsed',
+            'max_retries',
+            'retry_attempts',
+            'retried_jobs',
             'identifier',
         )
 
+    def get_condition_edges(self, obj):
+        return _workflow_node_condition_edges(obj)
+
     def get_related(self, obj):
         res = super(WorkflowJobNodeSerializer, self).get_related(obj)
         res['success_nodes'] = self.reverse('api:workflow_job_node_success_nodes_list', kwargs={'pk': obj.pk})
         res['failure_nodes'] = self.reverse('api:workflow_job_node_failure_nodes_list', kwargs={'pk': obj.pk})
         res['always_nodes'] = self.reverse('api:workflow_job_node_always_nodes_list', kwargs={'pk': obj.pk})
+        res['condition_nodes'] = self.reverse('api:workflow_job_node_condition_nodes_list', kwargs={'pk': obj.pk})
         if obj.unified_job_template:
             res['unified_job_template'] = obj.unified_job_template.get_absolute_url(self.context.get('request'))
         if obj.job:
@@ -4287,7 +4371,7 @@ def build_relational_field(self, field_name, relation_info):
 class WorkflowJobTemplateNodeCreateApprovalSerializer(BaseSerializer):
     class Meta:
         model = WorkflowApprovalTemplate
-        fields = ('timeout', 'name', 'description')
+        fields = ('timeout', 'name', 'description', 'context_template')
 
     def to_representation(self, obj):
         return {}

awx/api/templates/api/webhook_key_view.md

@@ -1,12 +1,15 @@
 Webhook Secret Key:
 
 Make a GET request to this resource to obtain the secret key for a job
-template or workflow job template configured to be triggered by
-webhook events.  The response will include the following fields:
+template, workflow job template or project configured to be triggered
+by webhook events.  The response will include the following fields:
 
 * `webhook_key`: Secret key that needs to be copied and added to the
-  webhook configuration of the service this template will be receiving
+  webhook configuration of the service this resource will be receiving
   webhook events from (string, read-only)
 
 Make an empty POST request to this resource to generate a new
 replacement `webhook_key`.
+
+A specific key can also be set by writing to the `webhook_key` field
+of the job template, workflow job template or project resource itself.

awx/api/urls/project.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.urls import path
+from django.urls import include, path
 
 from awx.api.views import (
     ProjectList,
@@ -47,6 +47,7 @@
     path('<int:pk>/object_roles/', ProjectObjectRolesList.as_view(), name='project_object_roles_list'),
     path('<int:pk>/access_list/', ProjectAccessList.as_view(), name='project_access_list'),
     path('<int:pk>/copy/', ProjectCopy.as_view(), name='project_copy'),
+    path('<int:pk>/', include('awx.api.urls.webhooks'), {'model_kwarg': 'projects'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/workflow_job_node.py

@@ -9,6 +9,7 @@
     WorkflowJobNodeSuccessNodesList,
     WorkflowJobNodeFailureNodesList,
     WorkflowJobNodeAlwaysNodesList,
+    WorkflowJobNodeConditionNodesList,
     WorkflowJobNodeCredentialsList,
     WorkflowJobNodeLabelsList,
     WorkflowJobNodeInstanceGroupsList,
@@ -20,6 +21,7 @@
     path('<int:pk>/success_nodes/', WorkflowJobNodeSuccessNodesList.as_view(), name='workflow_job_node_success_nodes_list'),
     path('<int:pk>/failure_nodes/', WorkflowJobNodeFailureNodesList.as_view(), name='workflow_job_node_failure_nodes_list'),
     path('<int:pk>/always_nodes/', WorkflowJobNodeAlwaysNodesList.as_view(), name='workflow_job_node_always_nodes_list'),
+    path('<int:pk>/condition_nodes/', WorkflowJobNodeConditionNodesList.as_view(), name='workflow_job_node_condition_nodes_list'),
     path('<int:pk>/credentials/', WorkflowJobNodeCredentialsList.as_view(), name='workflow_job_node_credentials_list'),
     path('<int:pk>/labels/', WorkflowJobNodeLabelsList.as_view(), name='workflow_job_node_labels_list'),
     path('<int:pk>/instance_groups/', WorkflowJobNodeInstanceGroupsList.as_view(), name='workflow_job_node_instance_groups_list'),

awx/api/urls/workflow_job_template_node.py

@@ -9,6 +9,7 @@
     WorkflowJobTemplateNodeSuccessNodesList,
     WorkflowJobTemplateNodeFailureNodesList,
     WorkflowJobTemplateNodeAlwaysNodesList,
+    WorkflowJobTemplateNodeConditionNodesList,
     WorkflowJobTemplateNodeCredentialsList,
     WorkflowJobTemplateNodeCreateApproval,
     WorkflowJobTemplateNodeLabelsList,
@@ -21,6 +22,7 @@
     path('<int:pk>/success_nodes/', WorkflowJobTemplateNodeSuccessNodesList.as_view(), name='workflow_job_template_node_success_nodes_list'),
     path('<int:pk>/failure_nodes/', WorkflowJobTemplateNodeFailureNodesList.as_view(), name='workflow_job_template_node_failure_nodes_list'),
     path('<int:pk>/always_nodes/', WorkflowJobTemplateNodeAlwaysNodesList.as_view(), name='workflow_job_template_node_always_nodes_list'),
+    path('<int:pk>/condition_nodes/', WorkflowJobTemplateNodeConditionNodesList.as_view(), name='workflow_job_template_node_condition_nodes_list'),
     path('<int:pk>/credentials/', WorkflowJobTemplateNodeCredentialsList.as_view(), name='workflow_job_template_node_credentials_list'),
     path('<int:pk>/labels/', WorkflowJobTemplateNodeLabelsList.as_view(), name='workflow_job_template_node_labels_list'),
     path('<int:pk>/instance_groups/', WorkflowJobTemplateNodeInstanceGroupsList.as_view(), name='workflow_job_template_node_instance_groups_list'),