diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 6944b663..76ca9c17 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3200,6 +3200,7 @@ class Meta: 'scm_branch', 'forks', 'limit', + 'job_slice_pinned_hosts', 'verbosity', 'extra_vars', 'job_tags', diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index a6f1e5a1..42a4151f 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3270,7 +3270,7 @@ def post(self, request, *args, **kwargs): if not jt: raise ParseError(_('Cannot relaunch slice workflow job orphaned from job template.')) elif getattr(obj.inventory, 'kind', None) != 'federated' and ( - not obj.inventory or min(obj.inventory.hosts.count(), jt.job_slice_count) != obj.workflow_nodes.count() + not obj.inventory or jt.get_effective_slice_ct({'inventory': obj.inventory}) != obj.workflow_nodes.count() ): raise ParseError(_('Cannot relaunch sliced workflow job after slice count has changed.')) if from_failed: diff --git a/awx/main/migrations/0204_job_slice_pinned_hosts.py b/awx/main/migrations/0204_job_slice_pinned_hosts.py new file mode 100644 index 00000000..4bdb3c7b --- /dev/null +++ b/awx/main/migrations/0204_job_slice_pinned_hosts.py @@ -0,0 +1,31 @@ +# Generated by Django 5.2.15 on 2026-07-05 00:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0203_workflow_node_retries'), + ] + + operations = [ + migrations.AddField( + model_name='job', + name='job_slice_pinned_hosts', + field=models.TextField( + blank=True, + default='', + help_text='Comma separated list of host names to include in every slice of a sliced job, in addition to the hosts of the slice itself. Useful when a play targets a coordinating host, such as localhost, that all slices depend on. Names are matched exactly against inventory hosts; groups and patterns are not supported. Has no effect unless the job is sliced.', + ), + ), + migrations.AddField( + model_name='jobtemplate', + name='job_slice_pinned_hosts', + field=models.TextField( + blank=True, + default='', + help_text='Comma separated list of host names to include in every slice of a sliced job, in addition to the hosts of the slice itself. Useful when a play targets a coordinating host, such as localhost, that all slices depend on. Names are matched exactly against inventory hosts; groups and patterns are not supported. Has no effect unless the job is sliced.', + ), + ), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index b3edb139..7f07b6c3 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -281,11 +281,15 @@ def parse_slice_params(slice_str): raise ParseError(_('Slice number must be 1 or higher.')) return (number, step) - def get_sliced_hosts(self, host_queryset, slice_number, slice_count): + def get_sliced_hosts(self, host_queryset, slice_number, slice_count, pinned_hosts=None): """ Returns a slice of Hosts given a slice number and total slice count, or the original queryset if slicing is not requested. + Hosts named in pinned_hosts are taken out of the round robin + distribution and included in every slice instead, so that plays which + target them (a coordinating host like localhost) run in all slices. + NOTE: If slicing is performed, this will return a List[Host] with the resulting slice. If slicing is not performed it will return the original queryset (not evaluating it or forcing it to a list). This @@ -297,10 +301,15 @@ def get_sliced_hosts(self, host_queryset, slice_number, slice_count): if slice_count > 1 and slice_number > 0: offset = slice_number - 1 + if pinned_hosts: + pinned = list(host_queryset.filter(name__in=pinned_hosts)) + if pinned: + sliced = list(host_queryset.exclude(name__in=pinned_hosts)[offset::slice_count]) + return sorted(sliced + pinned, key=lambda host: host.name) host_queryset = host_queryset[offset::slice_count] return host_queryset - def get_script_data(self, hostvars=False, towervars=False, show_all=False, slice_number=1, slice_count=1): + def get_script_data(self, hostvars=False, towervars=False, show_all=False, slice_number=1, slice_count=1, slice_pinned_hosts=None): hosts_kw = dict() if not show_all: hosts_kw['enabled'] = True @@ -308,7 +317,7 @@ def get_script_data(self, hostvars=False, towervars=False, show_all=False, slice if towervars: fetch_fields.append('enabled') host_queryset = self.hosts.filter(**hosts_kw).order_by('name').only(*fetch_fields) - hosts = self.get_sliced_hosts(host_queryset, slice_number, slice_count) + hosts = self.get_sliced_hosts(host_queryset, slice_number, slice_count, pinned_hosts=slice_pinned_hosts) data = dict() all_group = data.setdefault('all', dict()) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 7c77c85e..cb821fb5 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -110,6 +110,17 @@ class Meta: blank=True, default='', ) + job_slice_pinned_hosts = models.TextField( + blank=True, + default='', + help_text=_( + 'Comma separated list of host names to include in every slice of a sliced job, ' + 'in addition to the hosts of the slice itself. Useful when a play targets a ' + 'coordinating host, such as localhost, that all slices depend on. ' + 'Names are matched exactly against inventory hosts; groups and patterns are ' + 'not supported. Has no effect unless the job is sliced.' + ), + ) verbosity = models.PositiveIntegerField( choices=VERBOSITY_CHOICES, blank=True, @@ -162,6 +173,10 @@ class Meta: extra_vars_dict = VarsDictProperty('extra_vars', True) + @property + def job_slice_pinned_hosts_list(self): + return [name.strip() for name in self.job_slice_pinned_hosts.split(',') if name.strip()] + @property def machine_credential(self): return self.credentials.filter(credential_type__kind='ssh').first() @@ -374,7 +389,14 @@ def get_effective_slice_ct(self, kwargs): if self.ask_job_slice_count_on_launch and 'job_slice_count' in kwargs: actual_slice_count = kwargs['job_slice_count'] if actual_inventory: - return min(actual_slice_count, actual_inventory.hosts.count()) + host_count = actual_inventory.hosts.count() + pinned_names = self.job_slice_pinned_hosts_list + if pinned_names and host_count and actual_slice_count > 1: + # Pinned hosts are repeated in every slice, so they do not add + # to the number of slices worth spawning. + pinned_ct = actual_inventory.hosts.filter(name__in=pinned_names).count() + host_count = max(host_count - pinned_ct, 1) + return min(actual_slice_count, host_count) else: return actual_slice_count @@ -759,8 +781,12 @@ def _get_task_impact(self): if self.inventory is not None: count_hosts = self.inventory.total_hosts if self.job_slice_count > 1: + pinned_ct = self.inventory.hosts.filter(name__in=self.job_slice_pinned_hosts_list).count() + count_hosts = max(count_hosts - pinned_ct, 0) # Integer division intentional count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count + # pinned hosts run in every slice on top of its share + count_hosts += pinned_ct else: count_hosts = 5 if self.forks == 0 else self.forks return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1 @@ -922,7 +948,7 @@ def get_hosts_for_fact_cache(self): host_qs = self.inventory.hosts host_qs = host_qs.only(*HOST_FACTS_FIELDS) - host_qs = self.inventory.get_sliced_hosts(host_qs, self.job_slice_number, self.job_slice_count) + host_qs = self.inventory.get_sliced_hosts(host_qs, self.job_slice_number, self.job_slice_count, pinned_hosts=self.job_slice_pinned_hosts_list) return host_qs diff --git a/awx/main/tasks/jobs.py b/awx/main/tasks/jobs.py index 5ddaff93..86ef2a7f 100644 --- a/awx/main/tasks/jobs.py +++ b/awx/main/tasks/jobs.py @@ -327,6 +327,7 @@ def build_inventory(self, instance, private_data_dir): if hasattr(instance, 'job_slice_number'): script_params['slice_number'] = instance.job_slice_number script_params['slice_count'] = instance.job_slice_count + script_params['slice_pinned_hosts'] = instance.job_slice_pinned_hosts_list return self.write_inventory_file(instance.inventory, private_data_dir, 'hosts', script_params) diff --git a/awx/main/tests/functional/api/test_workflow_job.py b/awx/main/tests/functional/api/test_workflow_job.py index 6080cb35..717d4914 100644 --- a/awx/main/tests/functional/api/test_workflow_job.py +++ b/awx/main/tests/functional/api/test_workflow_job.py @@ -65,6 +65,22 @@ def test_workflow_job_relaunch_federated_inventory(organization, job_template, p post(url, user=admin_user, expect=201) +@pytest.mark.django_db +def test_workflow_job_relaunch_with_pinned_hosts(slice_jt_factory, post, admin_user): + """A sliced job with pinned hosts spawns fewer nodes than + min(hosts, slice_count); the relaunch check must apply the same + arithmetic instead of flagging a slice count change.""" + slice_jt = slice_jt_factory(3, jt_kwargs={'job_slice_pinned_hosts': 'foo0'}) + wfj = slice_jt.create_unified_job() + wfj.status = 'successful' + wfj.save() + assert wfj.is_sliced_job + assert wfj.workflow_nodes.count() == 2 + + url = reverse("api:workflow_job_relaunch", kwargs={'pk': wfj.pk}) + post(url, user=admin_user, expect=201) + + @pytest.mark.django_db @pytest.mark.parametrize( "is_admin, status", diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index ebd9424c..8728925c 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -91,6 +91,31 @@ def test_slice_subset_with_groups(self, inventory): data.pop('all') assert data == expected_data + def test_slice_subset_with_pinned_hosts(self, inventory): + for i in range(7): + inventory.hosts.create(name='host{}'.format(i)) + # host0 is pinned, the remaining 6 hosts are distributed round robin + for i in range(3): + data = inventory.get_script_data(slice_number=i + 1, slice_count=3, slice_pinned_hosts=['host0']) + assert data == {'all': {'hosts': ['host0', 'host{}'.format(i + 1), 'host{}'.format(i + 4)]}} + + def test_slice_pinned_host_keeps_group_membership(self, inventory): + group = inventory.groups.create(name='central_hosts') + group.hosts.add(inventory.hosts.create(name='central')) + for i in range(2): + inventory.hosts.create(name='host{}'.format(i)) + for i in range(2): + data = inventory.get_script_data(slice_number=i + 1, slice_count=2, slice_pinned_hosts=['central']) + assert data['central_hosts'] == {'hosts': ['central']} + assert data['all']['hosts'] == ['host{}'.format(i)] + + def test_slice_pinned_hosts_unknown_names_ignored(self, inventory): + for i in range(2): + inventory.hosts.create(name='host{}'.format(i)) + for i in range(2): + data = inventory.get_script_data(slice_number=i + 1, slice_count=2, slice_pinned_hosts=['does-not-exist']) + assert data == {'all': {'hosts': ['host{}'.format(i)]}} + @pytest.mark.django_db class TestActiveCount: diff --git a/awx/main/tests/functional/models/test_job.py b/awx/main/tests/functional/models/test_job.py index 8ada3576..704d549e 100644 --- a/awx/main/tests/functional/models/test_job.py +++ b/awx/main/tests/functional/models/test_job.py @@ -89,6 +89,57 @@ def test_slice_count_prompt_limited_by_inventory(self, job_template, inventory, unified_job = job_template.create_unified_job(job_slice_count=2) assert isinstance(unified_job, Job) + def test_effective_slice_count_with_pinned_hosts(self, job_template, inventory): + job_template.inventory = inventory + job_template.job_slice_count = 5 + job_template.job_slice_pinned_hosts = ' localhost, missing-host ,' + assert job_template.job_slice_pinned_hosts_list == ['localhost', 'missing-host'] + inventory.hosts.create(name='localhost') + for i in range(3): + inventory.hosts.create(name='foo{}'.format(i)) + # localhost is repeated in every slice, so only 3 hosts are worth distributing + assert job_template.get_effective_slice_ct({}) == 3 + + def test_effective_slice_count_all_hosts_pinned(self, job_template, inventory): + job_template.inventory = inventory + job_template.job_slice_count = 3 + job_template.job_slice_pinned_hosts = 'foo0,foo1' + for i in range(2): + inventory.hosts.create(name='foo{}'.format(i)) + # slicing is pointless when every host would end up in every slice + assert job_template.get_effective_slice_ct({}) == 1 + + def test_pinned_hosts_inherited_by_slice_jobs(self, slice_job_factory): + # factory creates hosts foo0..foo2 with slice count 3; foo0 pinned leaves + # two hosts to distribute, so only two slices get spawned + workflow_job = slice_job_factory(3, jt_kwargs={'job_slice_pinned_hosts': 'foo0'}, spawn=True) + assert workflow_job.workflow_nodes.count() == 2 + seen_hosts = [] + for node in workflow_job.workflow_nodes.all(): + job = node.job + assert job.job_slice_pinned_hosts == 'foo0' + # same call the task inventory build makes for the slice + data = job.inventory.get_script_data( + slice_number=job.job_slice_number, slice_count=job.job_slice_count, slice_pinned_hosts=job.job_slice_pinned_hosts_list + ) + hosts = data['all']['hosts'] + assert 'foo0' in hosts + assert len(hosts) == 2 + seen_hosts.extend(hosts) + # the two remaining hosts got distributed, one per slice + assert sorted(seen_hosts) == ['foo0', 'foo0', 'foo1', 'foo2'] + + def test_pinned_hosts_fact_cache_alignment(self, slice_job_factory): + # the fact cache host set of each slice must match the inventory the + # slice actually runs against + workflow_job = slice_job_factory(3, jt_kwargs={'job_slice_pinned_hosts': 'foo0', 'use_fact_cache': True}, spawn=True) + for node in workflow_job.workflow_nodes.all(): + job = node.job + data = job.inventory.get_script_data( + slice_number=job.job_slice_number, slice_count=job.job_slice_count, slice_pinned_hosts=job.job_slice_pinned_hosts_list + ) + assert sorted(host.name for host in job.get_hosts_for_fact_cache()) == data['all']['hosts'] + # --------------------------------------------------------------------------- # Federated inventory launch tests @@ -308,4 +359,3 @@ def test_complex_pattern_failsafe(self, organization): assert _federated_inventory_has_matching_hosts(inv, 'web:db') is True assert _federated_inventory_has_matching_hosts(inv, 'web&db') is True assert _federated_inventory_has_matching_hosts(inv, '!web') is True - diff --git a/awx/ui/src/screens/Template/JobTemplateDetail/JobTemplateDetail.js b/awx/ui/src/screens/Template/JobTemplateDetail/JobTemplateDetail.js index c804c852..3869894c 100644 --- a/awx/ui/src/screens/Template/JobTemplateDetail/JobTemplateDetail.js +++ b/awx/ui/src/screens/Template/JobTemplateDetail/JobTemplateDetail.js @@ -47,6 +47,7 @@ function JobTemplateDetail({ template }) { forks, host_config_key, job_slice_count, + job_slice_pinned_hosts, job_tags, job_type, name, @@ -289,6 +290,12 @@ function JobTemplateDetail({ template }) { dataCy="jt-detail-job-slice-count" helpText={helpText.jobSlicing} /> + {host_config_key && ( <> + {Number(jobSliceCountField.value) > 1 && ( + + )} ', () => { ); }); + test('job slice pinned hosts field only shows when slicing is enabled', async () => { + renderWithContexts( + + ); + await screen.findByRole('button', { name: 'Save' }); + + expect( + document.getElementById('template-job-slice-pinned-hosts') + ).toBeNull(); + + const sliceInput = document.getElementById('template-job-slicing'); + fireEvent.change(sliceInput, { + target: { value: '3', name: 'job_slice_count' }, + }); + + const pinnedInput = await waitFor(() => { + const el = document.getElementById('template-job-slice-pinned-hosts'); + if (!el) throw new Error('pinned hosts field not rendered yet'); + return el; + }); + fireEvent.change(pinnedInput, { + target: { value: 'localhost', name: 'job_slice_pinned_hosts' }, + }); + expect(pinnedInput.value).toBe('localhost'); + }); + test('webhooks and enable concurrent jobs functions properly', async () => { // WebhookSubForm reads the template id from useParams(), so mount the form // under a real v6 route whose concrete URL provides id=1. diff --git a/docs/job_slicing.md b/docs/job_slicing.md index f326ca0d..c5c51545 100644 --- a/docs/job_slicing.md +++ b/docs/job_slicing.md @@ -10,6 +10,19 @@ Job Slicing solves this problem by adding a Job Template field `job_slice_count` When jobs are sliced, they can run on any Ascender node; however, some may not run at the same time. Because of this, anything that relies on setting/sliced state (using modules such as `set_fact`) will not work as expected. It's reasonable to expect that not all jobs will actually run at the same time (*e.g.*, if there is not enough capacity in the system) +## Pinned Hosts + +Distributing the inventory evenly breaks playbooks in which a play targets a host that the rest of the run depends on. The typical example is a preparatory play against `localhost` that gathers data or builds configuration for every other host: after slicing, only the slice that happens to receive `localhost` runs that play, and the remaining slices fail or behave incorrectly. Until now the only options were to not slice such playbooks at all, or to restructure them so no play depends on another host. + +The Job Template field `job_slice_pinned_hosts` addresses exactly that problem. It takes a comma separated list of inventory host names that are excluded from the even distribution and included in **every** slice instead. Hosts named there keep their group memberships and variables in each slice. Matching is by exact host name: groups, globs and other `limit` style patterns are not supported. Names that do not match an inventory host are ignored, and the field has no effect when the job is not sliced. + +Note that the scope of the field is deliberately narrow: it only guarantees that the named hosts are part of every slice's inventory. It does not decide what runs where. That control stays in the playbook, as usual: each play's `hosts:` pattern determines what actually executes on which host, so a pinned host only runs the plays that target it, and the fleet plays keep running on each slice's share of the inventory. + +Because pinned hosts do not add to the work worth distributing, they are not counted when capping the slice count to the number of hosts: an inventory of 4 hosts with one pinned yields at most 3 slices. + +Keep in mind that plays targeting a pinned host run once per slice, concurrently. That is what you want for idempotent preparation on `localhost`, but be careful when pinning a shared machine, and note that `run_once` semantics become "once per slice". + + ## Simultaneous Execution Behavior By default, Job Templates aren't normally configured to execute simultaneously (`allow_simultaneous` must be checked). Slicing overrides this behavior and implies `allow_simultaneous`, even if that setting is not selected.