From 77daada214ec06f2b726b8d0a0e05df91d5cc63f Mon Sep 17 00:00:00 2001 From: muhammad-ammar Date: Fri, 30 Jan 2026 22:07:08 +0500 Subject: [PATCH] feat: construct and index spanish algolia objects for skills, jobs and industries test: improve algolia translation coverage fix: address review comments - external_id skill keys, early return, searchable metadata_language --- CHANGELOG.rst | 4 + taxonomy/__init__.py | 2 +- taxonomy/algolia/constants.py | 11 + taxonomy/algolia/utils.py | 406 ++++++++- taxonomy/tests/test_algolia_translations.py | 936 ++++++++++++++++++++ 5 files changed, 1350 insertions(+), 9 deletions(-) create mode 100644 taxonomy/tests/test_algolia_translations.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 04fa7cda..296c08f6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,10 @@ Change Log Unreleased +[3.0.0] - 2026-05-25 +--------------------- +* feat: construct and index Spanish Algolia objects for skills, jobs and industries + [2.4.0] - 2026-03-26 --------------------- * fix: bump lightcast version from 8.9 to 9.41 diff --git a/taxonomy/__init__.py b/taxonomy/__init__.py index 6ac9c2da..d8189320 100644 --- a/taxonomy/__init__.py +++ b/taxonomy/__init__.py @@ -15,4 +15,4 @@ # 2. MINOR version when you add functionality in a backwards compatible manner, and # 3. PATCH version when you make backwards compatible bug fixes. # More details can be found at https://semver.org/ -__version__ = '2.4.0' +__version__ = '3.0.0' diff --git a/taxonomy/algolia/constants.py b/taxonomy/algolia/constants.py index ca7d4793..d553c7a9 100644 --- a/taxonomy/algolia/constants.py +++ b/taxonomy/algolia/constants.py @@ -17,6 +17,13 @@ 'searchable(industry_names)', 'searchable(b2c_opt_in)', 'searchable(job_sources)', + 'filterOnly(metadata_language)', + ], + 'customRanking': [ + # Use a numeric priority field so English (0) always ranks above + # translated records (1), regardless of language code alphabetic order + # (e.g., Arabic 'ar' would otherwise rank higher than English 'en'). + 'asc(language_sort_priority)', ], } @@ -29,3 +36,7 @@ JOBS_TO_IGNORE = [ 'ET0000000000000000', # 'Unclassified' job ] + +# Default enabled locales for localized Algolia job records. +# Override per environment using Django setting: TAXONOMY_TRANSLATION_LOCALES. +TAXONOMY_TRANSLATION_LOCALES = ['es'] diff --git a/taxonomy/algolia/utils.py b/taxonomy/algolia/utils.py index c2143a95..33e556a5 100644 --- a/taxonomy/algolia/utils.py +++ b/taxonomy/algolia/utils.py @@ -15,9 +15,10 @@ EMBEDDED_OBJECT_LENGTH_CAP, JOBS_PAGE_SIZE, JOBS_TO_IGNORE, + TAXONOMY_TRANSLATION_LOCALES, ) from taxonomy.algolia.serializers import JobSerializer -from taxonomy.models import Industry, IndustryJobSkill, Job, JobSkills +from taxonomy.models import Industry, IndustryJobSkill, Job, JobSkills, Skill, TaxonomyTranslation LOGGER = logging.getLogger(__name__) @@ -52,13 +53,14 @@ def __exit__(self, *args, **kwargs): def index_jobs_data_in_algolia(): """ - Re-Index all jobs data to algolia. + Re-Index all jobs data to algolia with translations. - This function is responsible for - 1. Constructing a list of dicts containing all jobs present in the database. - 2. Re-Indexing all data in a single atomic operations with zero downtime. + This function is responsible for: + 1. Constructing a list of dicts containing all jobs present in the database (English). + 2. Creating localized variants for enabled languages (e.g., Spanish). + 3. Re-Indexing all data in a single atomic operation with zero downtime. - Note: We need to construct a list of all jobs in the form of a list and the send it all in a single attempt to + Note: We need to construct a list of all jobs in the form of a list and send it all in a single attempt to make the operation atomic and make sure there is no downtime. Paginating DB data and incrementally adding objects to the index will cause downtime equal to the amount of time it will take to run the command. """ @@ -67,14 +69,30 @@ def index_jobs_data_in_algolia(): api_key=settings.ALGOLIA.get('API_KEY'), index_name=settings.ALGOLIA.get('TAXONOMY_INDEX_NAME'), ) + LOGGER.info('[TAXONOMY] Resetting algolia index settings from code.') client.set_index_settings(ALGOLIA_JOBS_INDEX_SETTINGS) LOGGER.info('[TAXONOMY] Fetching Jobs data from the database.') - jobs_data = fetch_jobs_data() + english_jobs = fetch_jobs_data() LOGGER.info('[TAXONOMY] Jobs data successfully fetched from the database.') + LOGGER.info(f'[TAXONOMY] Total English job records: {len(english_jobs)}') + + translation_locales = getattr(settings, 'TAXONOMY_TRANSLATION_LOCALES', TAXONOMY_TRANSLATION_LOCALES) + + # Keep English jobs separate so localized variants are always translated + # from the original English records, not from previously translated records. + all_jobs = list(english_jobs) + for language in translation_locales: + LOGGER.info(f'[TAXONOMY] Creating {language} job records.') + localized_jobs = create_localized_job_records(english_jobs, language) + all_jobs.extend(localized_jobs) + LOGGER.info(f'[TAXONOMY] Added {len(localized_jobs)} {language} records.') + + LOGGER.info(f'[TAXONOMY] Total records (all languages): {len(all_jobs)}') + LOGGER.info('[TAXONOMY] Indexing Jobs data on algolia.') - client.replace_all_objects(jobs_data) + client.replace_all_objects(all_jobs) LOGGER.info('[TAXONOMY] Jobs data successfully indexed on algolia.') @@ -236,6 +254,372 @@ def get_job_ids(qs): return jobs +def build_name_translation_maps(language_code, prefetched_translations=None, scope=None): + """ + Build direct name→translation dictionaries using database queries. + + Uses database queries to map English entity names directly to translations, + avoiding the need for two-step lookups (name→id→translation). + + Args: + language_code: Target language (e.g., 'es') + prefetched_translations: Optional dict keyed by content_type containing + pre-fetched TaxonomyTranslation querysets/lists, e.g.:: + + { + 'job': [, ...], + 'skill': [, ...], + 'industry': [, ...], + } + + When provided, no additional TaxonomyTranslation queries are made. + Entity model queries are still used to map identifiers to English + names unless constrained by `scope`. + scope: Optional dict to limit model scans to relevant payload entities. + Supported keys: + job_external_ids, job_names, + skill_external_ids, skill_names, + industry_names + + Returns: + dict: { + 'job': {english_name: translated_name}, + 'skill': {english_name: translated_name}, + 'industry': {english_name: translated_name}, + } + """ + + LOGGER.info(f'[TAXONOMY] Building {language_code} translation maps from database.') + + if prefetched_translations is not None: + # Reuse pre-fetched data to avoid redundant DB queries. + job_trans_by_id = {t.external_id: t for t in prefetched_translations.get('job', [])} + skill_trans_by_id = {t.external_id: t for t in prefetched_translations.get('skill', [])} + industry_trans_by_id = {t.external_id: t for t in prefetched_translations.get('industry', [])} + else: + # Fetch all translations for the language in three targeted queries. + job_trans_by_id = { + t.external_id: t + for t in TaxonomyTranslation.objects.filter(content_type='job', language_code=language_code) + } + skill_trans_by_id = { + t.external_id: t + for t in TaxonomyTranslation.objects.filter(content_type='skill', language_code=language_code) + } + industry_trans_by_id = { + t.external_id: t + for t in TaxonomyTranslation.objects.filter(content_type='industry', language_code=language_code) + } + + if not (job_trans_by_id or skill_trans_by_id or industry_trans_by_id): + return { + 'job': {}, + 'skill': {}, + 'industry': {}, + } + + scope = scope or {} + job_external_ids = scope.get('job_external_ids') or set() + job_names = scope.get('job_names') or set() + skill_external_ids = scope.get('skill_external_ids') or set() + skill_names = scope.get('skill_names') or set() + industry_names = scope.get('industry_names') or set() + + # Job: English name to Translated name + job_translations = {} + jobs_qs = Job.objects.exclude(Q(name__isnull=True) | Q(external_id__in=JOBS_TO_IGNORE)) + if scope: + jobs_qs = jobs_qs.filter(Q(external_id__in=job_external_ids) | Q(name__in=job_names)) + for job in jobs_qs: + trans = job_trans_by_id.get(job.external_id) + if trans and trans.title: + job_translations[job.name] = trans.title + + # Skill: English name to Translated name + skill_translations = {} + skills_qs = Skill.objects.exclude(external_id__isnull=True) + if scope: + skills_qs = skills_qs.filter(Q(external_id__in=skill_external_ids) | Q(name__in=skill_names)) + for skill in skills_qs: + trans = skill_trans_by_id.get(skill.external_id) + if trans and trans.title: + # Prefer external_id-based lookup (stable/unique); also keep a name key + # for schemas that only include names (e.g., industry skill lists). + skill_translations[skill.external_id] = trans.title + if skill.name: + skill_translations[skill.name] = trans.title + + # Industry: English name to Translated name + industry_translations = {} + industries_qs = Industry.objects.exclude(code__isnull=True) + if scope and industry_names: + industries_qs = industries_qs.filter(name__in=industry_names) + for industry in industries_qs: + trans = industry_trans_by_id.get(str(industry.code)) + if trans and trans.title: + industry_translations[industry.name] = trans.title + + LOGGER.info( + f'[TAXONOMY] Built translation maps: {len(job_translations)} jobs, ' + f'{len(skill_translations)} skills, {len(industry_translations)} industries' + ) + + return { + 'job': job_translations, + 'skill': skill_translations, + 'industry': industry_translations, + } + + +def translate_skill_dict(skill, name_translation_maps): + """ + Translate a single skill dict using direct name lookup (keeps same schema). + + Args: + skill: Dict with skill data from JobSerializer + name_translation_maps: Direct name→translation dictionaries + + Returns: + Dict with translated skill data (same schema as input) + """ + skill_name = skill.get('name', '') + skill_external_id = skill.get('external_id') + + # Prefer external_id lookup (stable/unique); fall back to name lookup for + # schemas that only carry names (e.g., industry nested skill lists). + skill_map = name_translation_maps['skill'] + if skill_external_id and skill_external_id in skill_map: + translated_name = skill_map[skill_external_id] + else: + translated_name = skill_map.get(skill_name, skill_name) + + return { + **skill, # Copy all fields (significance, type_id, description, etc.) + 'name': translated_name, + } + + +def translate_industries_array(industries, name_translation_maps): + """ + Translate industries array with nested skills using direct name lookup (keeps same schema). + + Args: + industries: List of industry dicts from JobSerializer + name_translation_maps: Direct name→translation dictionaries + + Returns: + List of translated industry dicts (same schema as input) + """ + translated_industries = [] + + for industry in industries: + industry_name = industry.get('name', '') + + # Direct lookup: English industry name to Translated industry name + translated_industry_name = name_translation_maps['industry'].get(industry_name, industry_name) + + # Translate nested skills (they are plain strings in current schema) + translated_skills = [ + name_translation_maps['skill'].get(skill_name, skill_name) + for skill_name in industry.get('skills', []) + ] + + translated_industries.append({ + 'name': translated_industry_name, + 'skills': translated_skills # Keep as list of strings + }) + + return translated_industries + + +def translate_job_record(english_job, name_translation_maps, description_translation_maps, language_code): + """ + Translate a single job record using direct name lookups - creates duplicate with same schema. + + Args: + english_job: Dict with English job data (from JobSerializer) + name_translation_maps: Direct name to translated_name dictionaries + description_translation_maps: {content_type: {external_id: TaxonomyTranslation}} for descriptions + language_code: Target language code (e.g., 'es') + + Returns: + Dict with translated job data (SAME SCHEMA as English) + """ + external_id = english_job.get('external_id') + base_object_id = english_job.get('objectID') or f'job-{external_id}' + job_name = english_job.get('name', '') + + # Direct name translation + translated_job_name = name_translation_maps['job'].get(job_name, job_name) + + # Description requires external_id lookup (not included in name maps) + job_trans = description_translation_maps.get('job', {}).get(external_id) + translated_description = ( + job_trans.description if (job_trans and job_trans.description) + else english_job.get('description', '') + ) + + # Create localized copy with IDENTICAL schema + localized_job = { + # Metadata - change objectID and external_id for localized variant. + # external_id must be made unique per language to prevent Algolia's + # `distinct` setting (which groups on external_id) from hiding localized + # records behind the English record with the same external_id. + 'objectID': f'{base_object_id}-{language_code}', + 'id': english_job.get('id'), + 'external_id': f"{external_id}-{language_code}", + 'metadata_language': language_code, + 'language_sort_priority': 1, # Non-English records rank after English + + # Translated top-level fields + 'name': translated_job_name, + 'description': translated_description, + + # Translate skills array (keep same schema - dicts with name, description, etc.) + 'skills': [ + translate_skill_dict(skill, name_translation_maps) + for skill in english_job.get('skills', []) + ], + + # Job postings (no translation - copy as-is) + 'job_postings': list(english_job.get('job_postings', [])), + + # Translate industry_names (direct name lookup - list of strings) + 'industry_names': [ + name_translation_maps['industry'].get(ind_name, ind_name) + for ind_name in english_job.get('industry_names', []) + ], + + # Translate industries with nested skills (keep same schema) + 'industries': translate_industries_array( + english_job.get('industries', []), + name_translation_maps + ), + + # Translate similar_jobs (direct name lookup - list of strings) + 'similar_jobs': [ + name_translation_maps['job'].get(job_name, job_name) + for job_name in english_job.get('similar_jobs', []) + ], + + # Non-translatable fields (copy as-is) + 'b2c_opt_in': english_job.get('b2c_opt_in', False), + 'job_sources': english_job.get('job_sources', []), + } + + return localized_job + + +def create_localized_job_records(english_jobs, language_code): + """ + Create localized variants of English job records using direct name translation. + + This function: + 1. Builds direct name→translation dictionaries (English→Spanish) + 2. Fetches description translations separately (requires external_id) + 3. Creates duplicate job records with translated content using O(1) lookups + + Args: + english_jobs: List of serialized English job dicts + language_code: Target language code (e.g., 'es', 'fr', 'ar') + + Returns: + List of localized job dicts (SAME SCHEMA as English, different content) + """ + if not english_jobs: + return [] + + LOGGER.info(f'[TAXONOMY] Building {language_code} translation maps.') + + job_external_ids = {job.get('external_id') for job in english_jobs if job.get('external_id')} + job_names = { + job_name + for job in english_jobs + for job_name in [job.get('name'), *job.get('similar_jobs', [])] + if job_name + } + skill_external_ids = { + skill.get('external_id') + for job in english_jobs + for skill in job.get('skills', []) + if isinstance(skill, dict) and skill.get('external_id') + } + skill_names = { + skill_name + for job in english_jobs + for skill_name in [ + *(skill.get('name') for skill in job.get('skills', []) if isinstance(skill, dict)), + *(name for industry in job.get('industries', []) for name in industry.get('skills', [])), + ] + if skill_name + } + industry_names = { + industry_name + for job in english_jobs + for industry_name in [*job.get('industry_names', []), *(i.get('name') for i in job.get('industries', []))] + if industry_name + } + + extra_job_ids = set(Job.objects.filter(name__in=job_names).values_list('external_id', flat=True)) + job_external_ids.update(extra_job_ids) + industry_codes = { + str(code) + for code in Industry.objects.filter(name__in=industry_names).values_list('code', flat=True) + if code is not None + } + + all_translations = TaxonomyTranslation.objects.filter(language_code=language_code).filter( + Q(content_type='job', external_id__in=job_external_ids) + | Q(content_type='skill', external_id__in=skill_external_ids) + | Q(content_type='industry', external_id__in=industry_codes) + ) + + prefetched: dict = {'job': [], 'skill': [], 'industry': []} + description_translation_maps: dict = {'job': {}, 'skill': {}, 'industry': {}} + + for trans in all_translations: + ct = trans.content_type + if ct in prefetched: + prefetched[ct].append(trans) + description_translation_maps.setdefault(ct, {})[trans.external_id] = trans + + # Build direct name→translation maps reusing the pre-fetched data. + name_translation_maps = build_name_translation_maps( + language_code, + prefetched_translations=prefetched, + scope={ + 'job_external_ids': job_external_ids, + 'job_names': job_names, + 'skill_external_ids': skill_external_ids, + 'skill_names': skill_names, + 'industry_names': industry_names, + }, + ) + + LOGGER.info( + f'[TAXONOMY] Loaded {len(name_translation_maps["job"])} job name, ' + f'{len(name_translation_maps["skill"])} skill name, ' + f'{len(name_translation_maps["industry"])} industry name translations.' + ) + + # Create localized variants (duplicate records with translated content) + localized_jobs = [] + for idx, english_job in enumerate(english_jobs, 1): + if idx % 1000 == 0: + LOGGER.info(f'[TAXONOMY] Translated {idx}/{len(english_jobs)} jobs to {language_code}') + + localized_job = translate_job_record( + english_job, + name_translation_maps, + description_translation_maps, + language_code + ) + localized_jobs.append(localized_job) + + LOGGER.info(f'[TAXONOMY] Completed translating {len(localized_jobs)} jobs to {language_code}') + return localized_jobs + + def fetch_jobs_data(): """ Construct a list of all the jobs from the database. @@ -266,6 +650,12 @@ def fetch_jobs_data(): 'jobs_having_industry_skills': get_job_ids(IndustryJobSkill.get_whitelisted_job_skill_qs()), }, ) + # Add metadata fields to English records. + # language_sort_priority=0 ensures English always ranks first in + # Algolia results regardless of what language codes are in use. + for job_data in job_serializer.data: + job_data['metadata_language'] = 'en' + job_data['language_sort_priority'] = 0 jobs.extend(job_serializer.data) start += page_size diff --git a/taxonomy/tests/test_algolia_translations.py b/taxonomy/tests/test_algolia_translations.py new file mode 100644 index 00000000..44813e98 --- /dev/null +++ b/taxonomy/tests/test_algolia_translations.py @@ -0,0 +1,936 @@ +# -*- coding: utf-8 -*- +""" +Tests for Algolia translation utilities. +""" +import logging +from collections import deque +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +import taxonomy.algolia.utils as algolia_utils +from taxonomy.algolia.utils import ( + build_name_translation_maps, + create_localized_job_records, + fetch_jobs_data, + index_jobs_data_in_algolia, + translate_industries_array, + translate_job_record, + translate_skill_dict, +) +from taxonomy.models import Industry, IndustryJobSkill, Job, JobSkills, Skill, TaxonomyTranslation + + +@pytest.mark.django_db +class TestBuildNameTranslationMaps: + """Test building translation maps.""" + + @pytest.mark.parametrize('content_type,model_class,external_id,name,translation', [ + ('job', Job, 'ET123', 'Software Engineer', 'Ingeniero de Software'), + ('skill', Skill, 'ES123', 'Python', 'Python (Programación)'), + ('industry', Industry, '54', 'Information Technology', 'Tecnología de la Información'), + ]) + def test_builds_translations(self, content_type, model_class, external_id, name, translation): + """Test building translations for jobs, skills, and industries.""" + if model_class == Industry: + model_class.objects.create(code=external_id, name=name) + else: + model_class.objects.create(external_id=external_id, name=name) + + TaxonomyTranslation.objects.create( + external_id=external_id, + content_type=content_type, + language_code='es', + title=translation + ) + + maps = build_name_translation_maps('es') + + assert maps[content_type][name] == translation + + def test_skips_empty_translations(self): + """Test that empty translations are not included in maps.""" + job = Job.objects.create(external_id='ET123', name='Software Engineer') + TaxonomyTranslation.objects.create( + external_id='ET123', + content_type='job', + language_code='es', + title='' # Empty translation + ) + + maps = build_name_translation_maps('es') + + # Empty translation should not be in the map + assert 'Software Engineer' not in maps['job'] + + def test_returns_empty_maps_when_no_translations(self): + """Test returns empty dicts when no translations exist.""" + Job.objects.create(external_id='ET123', name='Software Engineer') + + maps = build_name_translation_maps('es') + + assert maps['job'] == {} + assert maps['skill'] == {} + assert maps['industry'] == {} + + def test_skips_empty_skill_translation_title(self): + """Test empty skill translation title is skipped (falls through condition).""" + Skill.objects.create(external_id='ES123', name='Python') + TaxonomyTranslation.objects.create( + external_id='ES123', + content_type='skill', + language_code='es', + title='', + ) + + maps = build_name_translation_maps('es') + + assert 'Python' not in maps['skill'] + + def test_accepts_prefetched_translations(self): + """Test that pre-fetched translations are used without issuing new DB queries.""" + Job.objects.create(external_id='ET123', name='Software Engineer') + trans = TaxonomyTranslation.objects.create( + external_id='ET123', + content_type='job', + language_code='es', + title='Ingeniero de Software', + ) + + prefetched = {'job': [trans], 'skill': [], 'industry': []} + maps = build_name_translation_maps('es', prefetched_translations=prefetched) + + assert maps['job']['Software Engineer'] == 'Ingeniero de Software' + + +class TestTranslateSkillDict: + """Test skill dict translation.""" + + def test_translates_skill_name(self): + """Test skill name is translated and all fields are preserved.""" + skill = { + 'name': 'Python', + 'description': 'Programming language', + 'significance': 85, + 'type_id': 'ST1' + } + name_maps = { + 'skill': {'Python': 'Python (Lenguaje)'} + } + + result = translate_skill_dict(skill, name_maps) + + assert result['name'] == 'Python (Lenguaje)' + assert result['description'] == 'Programming language' + assert result['significance'] == 85 + assert result['type_id'] == 'ST1' + + def test_fallback_when_translation_missing(self): + """Test falls back to English when translation not found.""" + skill = {'name': 'JavaScript'} + name_maps = {'skill': {'Python': 'Python (ES)'}} + + result = translate_skill_dict(skill, name_maps) + + assert result['name'] == 'JavaScript' + + +class TestTranslateIndustriesArray: + """Test industries array translation.""" + + def test_translates_industry_names(self): + """Test industry names are translated.""" + industries = [ + { + 'name': 'Information Technology', + 'skills': [] + } + ] + name_maps = { + 'industry': {'Information Technology': 'Tecnología de la Información'}, + 'skill': {} + } + + result = translate_industries_array(industries, name_maps) + + assert result[0]['name'] == 'Tecnología de la Información' + + def test_translates_nested_skills(self): + """Test nested skills are translated.""" + industries = [ + { + 'name': 'IT', + 'skills': ['Python', 'Java', 'Cloud Computing'] + } + ] + name_maps = { + 'industry': {'IT': 'TI'}, + 'skill': { + 'Python': 'Python (Programación)', + 'Cloud Computing': 'Computación en la Nube' + } + } + + result = translate_industries_array(industries, name_maps) + + assert result[0]['name'] == 'TI' + assert result[0]['skills'][0] == 'Python (Programación)' + assert result[0]['skills'][1] == 'Java' # Fallback + assert result[0]['skills'][2] == 'Computación en la Nube' + + def test_handles_empty_industries(self): + """Test handles empty industries list.""" + result = translate_industries_array([], {'industry': {}, 'skill': {}}) + + assert result == [] + + +@pytest.mark.django_db +class TestTranslateJobRecord: + """Test job record translation.""" + + def test_translates_job_name(self): + """Test job name is translated.""" + Job.objects.create(external_id='ET123', name='Software Engineer') + TaxonomyTranslation.objects.create( + external_id='ET123', + content_type='job', + language_code='es', + title='Ingeniero de Software', + description='Desarrolla software' + ) + + english_job = { + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Software Engineer', + 'description': 'Develops software', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': True, + 'job_sources': ['course_skill'] + } + + name_maps = {'job': {'Software Engineer': 'Ingeniero de Software'}, 'skill': {}, 'industry': {}} + desc_maps = { + 'job': { + 'ET123': TaxonomyTranslation.objects.get(external_id='ET123') + }, + 'skill': {}, + 'industry': {} + } + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + assert result['objectID'] == 'job-ET123-es' + assert result['name'] == 'Ingeniero de Software' + assert result['description'] == 'Desarrolla software' + assert result['metadata_language'] == 'es' + + def test_translates_all_nested_fields(self): + """Test all nested arrays are translated.""" + english_job = { + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': 'Desc', + 'skills': [ + {'name': 'Python', 'significance': 90} + ], + 'job_postings': [{'id': 1}], + 'industry_names': ['IT', 'Software'], + 'industries': [ + {'name': 'IT', 'skills': ['Cloud']} + ], + 'similar_jobs': ['Senior Engineer', 'Architect'], + 'b2c_opt_in': False, + 'job_sources': ['job_skill'] + } + + name_maps = { + 'job': {'Engineer': 'Ingeniero', 'Senior Engineer': 'Ingeniero Senior'}, + 'skill': {'Python': 'Python (ES)', 'Cloud': 'Nube'}, + 'industry': {'IT': 'TI', 'Software': 'Software'} + } + desc_maps = {'job': {}, 'skill': {}, 'industry': {}} + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + # Check skills translated + assert result['skills'][0]['name'] == 'Python (ES)' + assert result['skills'][0]['significance'] == 90 + + # Check industry_names translated + assert result['industry_names'] == ['TI', 'Software'] + + # Check industries with nested skills translated + assert result['industries'][0]['name'] == 'TI' + assert result['industries'][0]['skills'] == ['Nube'] + + # Check similar_jobs translated + assert result['similar_jobs'][0] == 'Ingeniero Senior' + assert result['similar_jobs'][1] == 'Architect' # Fallback + + def test_preserves_non_translatable_fields(self): + """Test non-translatable fields are preserved.""" + english_job = { + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': '', + 'skills': [], + 'job_postings': [{'id': 1, 'url': 'http://example.com'}], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': True, + 'job_sources': ['course_skill', 'job_skill'] + } + + name_maps = {'job': {}, 'skill': {}, 'industry': {}} + desc_maps = {'job': {}, 'skill': {}, 'industry': {}} + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + assert result['job_postings'] == [{'id': 1, 'url': 'http://example.com'}] + assert result['b2c_opt_in'] is True + assert result['job_sources'] == ['course_skill', 'job_skill'] + assert result['id'] == 1 + # external_id is made composite so Algolia's `distinct` setting does not + # collapse localized records onto the English record with the same id. + assert result['external_id'] == 'ET123-es' + + def test_localized_record_has_language_sort_priority_one(self): + """Test localized records have language_sort_priority=1.""" + english_job = { + 'objectID': 'job-ET123', 'id': 1, 'external_id': 'ET123', + 'name': 'Engineer', 'description': '', 'skills': [], + 'job_postings': [], 'industry_names': [], 'industries': [], + 'similar_jobs': [], 'b2c_opt_in': False, 'job_sources': [] + } + name_maps = {'job': {}, 'skill': {}, 'industry': {}} + desc_maps = {'job': {}, 'skill': {}, 'industry': {}} + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + assert result['language_sort_priority'] == 1 + + def test_fallback_to_english_when_no_description(self): + """Test falls back to English description when translation empty.""" + english_job = { + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': 'English description', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [] + } + + name_maps = {'job': {}, 'skill': {}, 'industry': {}} + desc_maps = {'job': {}, 'skill': {}, 'industry': {}} + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + assert result['description'] == 'English description' + + def test_uses_existing_object_id_format(self): + """Localized objectID should preserve existing serializer format.""" + english_job = { + 'objectID': 'custom-prefix-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': '', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [] + } + name_maps = {'job': {}, 'skill': {}, 'industry': {}} + desc_maps = {'job': {}, 'skill': {}, 'industry': {}} + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + assert result['objectID'] == 'custom-prefix-ET123-es' + + def test_job_postings_list_is_shallow_copied(self): + """Localized record should not share the same postings list reference.""" + english_postings = [{'id': 1}] + english_job = { + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': '', + 'skills': [], + 'job_postings': english_postings, + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [] + } + name_maps = {'job': {}, 'skill': {}, 'industry': {}} + desc_maps = {'job': {}, 'skill': {}, 'industry': {}} + + result = translate_job_record(english_job, name_maps, desc_maps, 'es') + + assert result['job_postings'] == english_postings + assert result['job_postings'] is not english_postings + + +@pytest.mark.django_db +class TestCreateLocalizedJobRecords: + """Test creating localized job records.""" + + def test_creates_spanish_records(self): + """Test creates Spanish variant of English jobs.""" + job = Job.objects.create(external_id='ET123', name='Engineer') + TaxonomyTranslation.objects.create( + external_id='ET123', + content_type='job', + language_code='es', + title='Ingeniero', + description='Descripción' + ) + + english_jobs = [{ + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': 'Description', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [] + }] + + spanish_jobs = create_localized_job_records(english_jobs, 'es') + + assert len(spanish_jobs) == 1 + assert spanish_jobs[0]['objectID'] == 'job-ET123-es' + assert spanish_jobs[0]['external_id'] == 'ET123-es' + assert spanish_jobs[0]['name'] == 'Ingeniero' + assert spanish_jobs[0]['description'] == 'Descripción' + assert spanish_jobs[0]['metadata_language'] == 'es' + assert spanish_jobs[0]['language_sort_priority'] == 1 + + def test_creates_multiple_records(self): + """Test creates translations for multiple jobs.""" + Job.objects.create(external_id='ET1', name='Engineer') + Job.objects.create(external_id='ET2', name='Designer') + + TaxonomyTranslation.objects.create( + external_id='ET1', content_type='job', language_code='es', title='Ingeniero' + ) + TaxonomyTranslation.objects.create( + external_id='ET2', content_type='job', language_code='es', title='Diseñador' + ) + + english_jobs = [ + { + 'objectID': 'job-ET1', 'id': 1, 'external_id': 'ET1', 'name': 'Engineer', + 'description': '', 'skills': [], 'job_postings': [], 'industry_names': [], + 'industries': [], 'similar_jobs': [], 'b2c_opt_in': False, 'job_sources': [] + }, + { + 'objectID': 'job-ET2', 'id': 2, 'external_id': 'ET2', 'name': 'Designer', + 'description': '', 'skills': [], 'job_postings': [], 'industry_names': [], + 'industries': [], 'similar_jobs': [], 'b2c_opt_in': False, 'job_sources': [] + } + ] + + spanish_jobs = create_localized_job_records(english_jobs, 'es') + + assert len(spanish_jobs) == 2 + assert spanish_jobs[0]['name'] == 'Ingeniero' + assert spanish_jobs[0]['metadata_language'] == 'es' + assert spanish_jobs[1]['name'] == 'Diseñador' + assert spanish_jobs[1]['metadata_language'] == 'es' + + def test_handles_partial_translations(self): + """Test gracefully handles missing translations.""" + Job.objects.create(external_id='ET1', name='Engineer') + TaxonomyTranslation.objects.create( + external_id='ET1', content_type='job', language_code='es', title='Ingeniero' + ) + + english_jobs = [{ + 'objectID': 'job-ET1', 'id': 1, 'external_id': 'ET1', 'name': 'Engineer', + 'description': '', 'skills': [{'name': 'Python'}], 'job_postings': [], + 'industry_names': ['IT'], 'industries': [], 'similar_jobs': ['Architect'], + 'b2c_opt_in': False, 'job_sources': [] + }] + + # No skill/industry translations + spanish_jobs = create_localized_job_records(english_jobs, 'es') + + # Should fall back to English for missing translations + assert spanish_jobs[0]['name'] == 'Ingeniero' # Translated + assert spanish_jobs[0]['skills'][0]['name'] == 'Python' # Fallback + assert spanish_jobs[0]['industry_names'][0] == 'IT' # Fallback + assert spanish_jobs[0]['similar_jobs'][0] == 'Architect' # Fallback + assert spanish_jobs[0]['metadata_language'] == 'es' + + def test_adds_metadata_language_field(self): + """Test metadata_language field is added to translated jobs.""" + job = Job.objects.create(external_id='ET123', name='Engineer') + + english_jobs = [{ + 'objectID': 'job-ET123', + 'id': 1, + 'external_id': 'ET123', + 'name': 'Engineer', + 'description': 'Desc', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [] + }] + + spanish_jobs = create_localized_job_records(english_jobs, 'es') + + assert 'metadata_language' in spanish_jobs[0] + assert spanish_jobs[0]['metadata_language'] == 'es' + + def test_returns_empty_list_when_no_jobs(self): + """Test returns empty list when no jobs provided.""" + result = create_localized_job_records([], 'es') + + assert result == [] + + def test_logs_progress_every_thousand_records(self, caplog, monkeypatch): + """Test progress log branch is hit when processing every 1000th record.""" + english_jobs = [ + { + 'objectID': f'job-{idx}', + 'id': idx, + 'external_id': f'ET{idx}', + 'name': 'Engineer', + 'description': '', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [], + } + for idx in range(1, 1001) + ] + + monkeypatch.setattr( + algolia_utils, + 'translate_job_record', + lambda english_job, *_: { + **english_job, + 'metadata_language': 'es', + } + ) + + caplog.set_level(logging.INFO) + result = create_localized_job_records(english_jobs, 'es') + + assert len(result) == 1000 + assert any('Translated 1000/1000 jobs to es' in record.message for record in caplog.records) + + def test_ignores_unexpected_translation_content_type(self, monkeypatch): + """Test unknown translation content types do not break localization flow.""" + english_jobs = [{ + 'objectID': 'job-ET1', + 'id': 1, + 'external_id': 'ET1', + 'name': 'Engineer', + 'description': '', + 'skills': [], + 'job_postings': [], + 'industry_names': [], + 'industries': [], + 'similar_jobs': [], + 'b2c_opt_in': False, + 'job_sources': [], + }] + + # Include a translation record with an unexpected content_type so that + # `if ct in prefetched` follows the False branch. + unknown_trans = SimpleNamespace(content_type='unknown', external_id='X1') + + class FakeTranslationQuerySet: + def __init__(self, values): + self.values = values + + def filter(self, *args, **kwargs): + return self + + def __iter__(self): + return iter(self.values) + + monkeypatch.setattr( + TaxonomyTranslation.objects, + 'filter', + lambda *args, **kwargs: FakeTranslationQuerySet([unknown_trans]), + ) + + monkeypatch.setattr( + algolia_utils, + 'build_name_translation_maps', + lambda *_args, **_kwargs: {'job': {}, 'skill': {}, 'industry': {}}, + ) + monkeypatch.setattr( + algolia_utils, + 'translate_job_record', + lambda english_job, *_: {**english_job, 'metadata_language': 'es'}, + ) + + localized = create_localized_job_records(english_jobs, 'es') + + assert len(localized) == 1 + assert localized[0]['metadata_language'] == 'es' + + +@pytest.mark.django_db +class TestIndexJobsDataInAlgolia: + """Tests for full index build flow with localized records.""" + + def test_indexes_english_and_localized_jobs(self, monkeypatch): + """Test indexing appends localized records for each configured language.""" + client = MagicMock() + monkeypatch.setattr(algolia_utils, 'AlgoliaClient', MagicMock(return_value=client)) + + english_jobs = [{'objectID': 'job-ET1', 'name': 'Engineer', 'metadata_language': 'en'}] + monkeypatch.setattr(algolia_utils, 'fetch_jobs_data', lambda: list(english_jobs)) + monkeypatch.setattr(algolia_utils, 'TAXONOMY_TRANSLATION_LOCALES', ['es', 'fr']) + + def _create_localized(jobs_data, language_code): + return [{'objectID': f'job-ET1-{language_code}', 'name': 'Engineer', 'metadata_language': language_code}] + + monkeypatch.setattr(algolia_utils, 'create_localized_job_records', _create_localized) + + index_jobs_data_in_algolia() + + client.set_index_settings.assert_called_once() + indexed_objects = client.replace_all_objects.call_args[0][0] + assert len(indexed_objects) == 3 + assert {obj['metadata_language'] for obj in indexed_objects} == {'en', 'es', 'fr'} + + def test_only_english_records_passed_to_create_localized(self, monkeypatch): + """Test create_localized_job_records always receives only English records. + + Regression test: previously jobs_data was mutated in-place so the second + language iteration would receive English + first-language records, causing + translated records to be re-translated. + """ + client = MagicMock() + monkeypatch.setattr(algolia_utils, 'AlgoliaClient', MagicMock(return_value=client)) + + english_jobs = [ + {'objectID': 'job-ET1', 'name': 'Engineer', 'metadata_language': 'en'}, + ] + monkeypatch.setattr(algolia_utils, 'fetch_jobs_data', lambda: list(english_jobs)) + monkeypatch.setattr(algolia_utils, 'TAXONOMY_TRANSLATION_LOCALES', ['es', 'fr']) + + received_inputs = [] + + def _create_localized(jobs_data, language_code): + received_inputs.append((language_code, list(jobs_data))) + return [{'objectID': f'job-ET1-{language_code}', 'metadata_language': language_code}] + + monkeypatch.setattr(algolia_utils, 'create_localized_job_records', _create_localized) + + index_jobs_data_in_algolia() + + # Both calls must have received only the single English record. + for lang, jobs_passed in received_inputs: + assert len(jobs_passed) == 1, ( + f'create_localized_job_records for {lang!r} received {len(jobs_passed)} records ' + f'instead of 1 English record; translated records may be leaking into subsequent iterations.' + ) + assert jobs_passed[0]['metadata_language'] == 'en' + + def test_uses_settings_locales_when_available(self, monkeypatch): + """Runtime settings should override module-level default locales.""" + client = MagicMock() + monkeypatch.setattr(algolia_utils, 'AlgoliaClient', MagicMock(return_value=client)) + monkeypatch.setattr(algolia_utils, 'fetch_jobs_data', lambda: [{'objectID': 'job-ET1', 'metadata_language': 'en'}]) + monkeypatch.setattr(algolia_utils, 'TAXONOMY_TRANSLATION_LOCALES', ['fr']) + monkeypatch.setattr(algolia_utils.settings, 'TAXONOMY_TRANSLATION_LOCALES', ['es'], raising=False) + + received_languages = [] + + def _create_localized(_jobs_data, language_code): + received_languages.append(language_code) + return [{'objectID': f'job-ET1-{language_code}', 'metadata_language': language_code}] + + monkeypatch.setattr(algolia_utils, 'create_localized_job_records', _create_localized) + + index_jobs_data_in_algolia() + + assert received_languages == ['es'] + + +@pytest.mark.django_db +class TestFetchJobsData: + """Tests for english jobs serialization payload.""" + + def test_adds_metadata_language_to_serialized_jobs(self, monkeypatch): + """Test serialized jobs include metadata_language='en'.""" + Job.objects.create(external_id='ET1', name='Engineer') + + monkeypatch.setattr(algolia_utils, 'fetch_and_combine_job_details', lambda _qs: {}) + monkeypatch.setattr(algolia_utils, 'combine_industry_skills', lambda: {}) + monkeypatch.setattr(algolia_utils, 'get_job_ids', lambda _qs: set()) + monkeypatch.setattr(JobSkills, 'get_whitelisted_job_skill_qs', classmethod(lambda cls: JobSkills.objects.none())) + monkeypatch.setattr( + IndustryJobSkill, + 'get_whitelisted_job_skill_qs', + classmethod(lambda cls: IndustryJobSkill.objects.none()) + ) + + class DummySerializer: + """Serializer stub for deterministic test payload.""" + + def __init__(self, *args, **kwargs): + self.data = [{'objectID': 'job-ET1', 'name': 'Engineer'}] + + monkeypatch.setattr(algolia_utils, 'JobSerializer', DummySerializer) + + jobs = fetch_jobs_data() + + assert jobs == [ + {'objectID': 'job-ET1', 'name': 'Engineer', 'metadata_language': 'en', 'language_sort_priority': 0} + ] + + +class TestTranslateSkillDictExternalId: + """Extra tests for external_id-first skill translation behavior.""" + + def test_prefers_external_id_translation_over_name(self): + """When both keys exist, external_id mapping should win.""" + skill = {'external_id': 'ES123', 'name': 'Python'} + name_maps = {'skill': {'ES123': 'Python (ID)', 'Python': 'Python (Name)'}} + + result = translate_skill_dict(skill, name_maps) + + assert result['name'] == 'Python (ID)' + + +class TestBuildNameTranslationMapsExternalId: + """Extra tests for branch coverage in name map builder.""" + + def test_skill_external_id_map_when_name_missing(self, monkeypatch): + """Build maps should include external_id key even if name is falsey.""" + translation = SimpleNamespace(external_id='ES123', title='Python (ID)') + prefetched = {'job': [], 'skill': [translation], 'industry': []} + + monkeypatch.setattr(Job.objects, 'exclude', lambda *args, **kwargs: []) + monkeypatch.setattr(Industry.objects, 'exclude', lambda *args, **kwargs: []) + monkeypatch.setattr( + Skill.objects, + 'exclude', + lambda *args, **kwargs: [SimpleNamespace(external_id='ES123', name='')], + ) + + maps = build_name_translation_maps('es', prefetched_translations=prefetched) + + assert maps['skill']['ES123'] == 'Python (ID)' + assert '' not in maps['skill'] + + +class TestAlgoliaCoreUtils: + """Coverage tests for core helper utilities in algolia utils.""" + + def test_calculate_jaccard_similarity_for_empty_sets(self): + """Empty sets should return 0.0 via ZeroDivisionError path.""" + assert algolia_utils.calculate_jaccard_similarity(set(), set()) == 0.0 + + def test_calculate_jaccard_similarity_for_non_empty_sets(self): + """Non-empty sets should compute Jaccard similarity.""" + result = algolia_utils.calculate_jaccard_similarity({'a', 'b'}, {'b', 'c'}) + assert result == 1 / 3 + + def test_insert_item_in_ordered_queue_replaces_tail_when_full(self): + """Better item in full queue should be inserted and tail popped.""" + queue = deque([5, 3, 1], maxlen=3) + + algolia_utils.insert_item_in_ordered_queue(queue, 4) + + assert list(queue) == [5, 4, 3] + + def test_insert_item_in_ordered_queue_appends_when_space(self): + """If no insertion point but queue has room, item should append.""" + queue = deque([5, 4], maxlen=3) + + algolia_utils.insert_item_in_ordered_queue(queue, 1) + + assert list(queue) == [5, 4, 1] + + def test_insert_item_in_ordered_queue_noop_when_full_and_low_priority(self): + """If full and item is lower than all entries, queue is unchanged.""" + queue = deque([5, 4, 3], maxlen=3) + + algolia_utils.insert_item_in_ordered_queue(queue, 1) + + assert list(queue) == [5, 4, 3] + + def test_calculate_job_recommendations_adds_similar_jobs(self): + """Similar jobs list should be present, bounded to 3, and exclude self.""" + jobs_data = { + 'Job A': {'skills': {'python', 'sql'}}, + 'Job B': {'skills': {'python', 'sql', 'aws'}}, + 'Job C': {'skills': {'java'}}, + 'Job D': {'skills': {'python'}}, + } + + result = algolia_utils.calculate_job_recommendations(jobs_data) + + assert 'similar_jobs' in result['Job A'] + assert 'Job A' not in result['Job A']['similar_jobs'] + assert len(result['Job A']['similar_jobs']) <= 3 + + def test_calculate_job_skills_builds_skill_set_per_job(self, monkeypatch): + """Job skill names should be converted to sets per job.""" + jobs = [SimpleNamespace(name='Job A'), SimpleNamespace(name='Job B')] + + class FakeJobsQuerySet: + def all(self): + return jobs + + class FakeSkillQuerySet: + def __init__(self): + self.current_job = None + + def filter(self, job): + self.current_job = job + return self + + def values_list(self, *_args, **_kwargs): + mapping = { + 'Job A': ['Python', 'SQL'], + 'Job B': ['Java'], + } + return mapping[self.current_job.name] + + monkeypatch.setattr( + JobSkills, + 'get_whitelisted_job_skill_qs', + classmethod(lambda _cls: FakeSkillQuerySet()), + ) + + result = algolia_utils.calculate_job_skills(FakeJobsQuerySet()) + + assert result == { + 'Job A': {'skills': {'Python', 'SQL'}}, + 'Job B': {'skills': {'Java'}}, + } + + def test_fetch_and_combine_job_details_calls_both_stages(self, monkeypatch): + """Pipeline should call calculate_job_skills then recommendations.""" + marker_qs = object() + skills_data = {'Job A': {'skills': {'Python'}}} + final_data = {'Job A': {'skills': {'Python'}, 'similar_jobs': []}} + + monkeypatch.setattr(algolia_utils, 'calculate_job_skills', lambda qs: skills_data if qs is marker_qs else {}) + monkeypatch.setattr( + algolia_utils, + 'calculate_job_recommendations', + lambda jobs_data: final_data if jobs_data is skills_data else {}, + ) + + result = algolia_utils.fetch_and_combine_job_details(marker_qs) + + assert result == final_data + + def test_combine_industry_skills_constructs_mapping(self, monkeypatch): + """Industry-to-skills mapping should be built from queryset chains.""" + industries = [SimpleNamespace(name='IT'), SimpleNamespace(name='Finance')] + + class FakeIndustrySkillQS: + def __init__(self): + self.current_industry = None + self.mapping = { + 'IT': ['Python', 'Cloud'], + 'Finance': ['Excel'], + } + + def filter(self, industry): + self.current_industry = industry.name + return self + + def values_list(self, *_args, **_kwargs): + return self + + def annotate(self, **_kwargs): + return self + + def order_by(self, *_args, **_kwargs): + return self + + def distinct(self): + return self + + def __getitem__(self, item): + return self.mapping[self.current_industry][item] + + monkeypatch.setattr(Industry.objects, 'all', lambda: industries) + monkeypatch.setattr( + IndustryJobSkill, + 'get_whitelisted_job_skill_qs', + classmethod(lambda _cls: FakeIndustrySkillQS()), + ) + + result = algolia_utils.combine_industry_skills() + + assert result == { + 'IT': ['Python', 'Cloud'], + 'Finance': ['Excel'], + } + + def test_get_job_ids_collects_ids_from_batches(self): + """Batched job id collection should aggregate all values.""" + + class FakeSlice: + def __init__(self, values): + self.values = values + + def exists(self): + return bool(self.values) + + def values_list(self, *_args, **_kwargs): + return self.values + + class FakeQuerySet: + def __init__(self, values): + self.values = values + + def all(self): + return self + + def __getitem__(self, item): + return FakeSlice(self.values[item.start:item.stop]) + + result = algolia_utils.get_job_ids(FakeQuerySet([1, 2, 3])) + + assert result == {1, 2, 3}