Prod release: support unicode in data, load local jsPsych (dev), optimize email task

accounts/queries.py

@@ -3,7 +3,7 @@
 import ast
 import operator
 from datetime import date
-from functools import reduce
+from functools import lru_cache, reduce
 from itertools import chain
 
 from django.db import models
@@ -97,9 +97,14 @@ def age_range_eligibility_for_study(child_age_range, study) -> bool:
 
 
 def get_child_eligibility_for_study(child_obj, study_obj):
+    # Order matters for performance: this runs once per child-study pair across the
+    # full announcement-email scan. The age-range check is pure Python (no DB), so it
+    # goes first to short-circuit the majority of ineligible pairs before we incur the
+    # DB queries in get_child_participation_eligibility or the expression evaluation in
+    # get_child_eligibility.
     return (
-        get_child_participation_eligibility(child_obj, study_obj)
-        and _child_in_age_range_for_study(child_obj, study_obj)
+        _child_in_age_range_for_study(child_obj, study_obj)
+        and get_child_participation_eligibility(child_obj, study_obj)
         and get_child_eligibility(child_obj, study_obj.criteria_expression)
     )
 
@@ -215,9 +220,18 @@ def get_child_eligibility(child_obj, criteria_expr):
         return True
 
 
+@lru_cache(maxsize=1024)
 def compile_expression(boolean_algebra_expression: str):
     """Compiles a boolean algebra expression into a python function.
 
+    The result is cached (keyed on the expression string) because a criteria
+    expression only depends on the study, not the child. Without this, the
+    announcement-email scan re-parses and re-compiles the same expression once
+    per child-study pair. The number of distinct expressions is bounded by the
+    number of distinct study criteria (i.e. number of studies), and in reality
+    there is substantial overlap in expressions across studies, so the cache
+    stays small.
+
     Args:
         boolean_algebra_expression: a string boolean algebra expression.
 

env_dist

@@ -52,6 +52,32 @@ JSPSYCH_S3_ACCESS_KEY_ID=
 JSPSYCH_S3_SECRET_ACCESS_KEY=
 JSPSYCH_S3_BUCKET=
 
+# Serve the CHS jsPsych (@lookit/*) packages from locally running dev builds instead of
+# the published unpkg URLs stored in the database. When enabled, the jsPsych study runner
+# swaps in local dev-server URLs (built in project/settings.py) and drops SRI integrity
+# for them. Requires serving each package locally (npm run dev -w @lookit/* in each 
+# lookit-jspsych package, or honcho start if used to serve multiple packages; 
+# see https://github.com/lookit/lookit-jspsych#serve-multiple-packages).
+# Uncomment to enable, then recreate the web container so the new .env value is
+# picked up (`docker compose up -d web` -- a plain `restart` does NOT re-read this file).
+# To use the published/unpkg URLs from the DB, either comment this out or set it to a
+# falsy value (False/false/0/no/off/empty all disable it).
+# JSPSYCH_LOCAL_PLUGINS=True
+#
+# The dev-server ports below match the defaults set in settings.py and rollup.config.dev.mjs
+# files for lookit-jspsych packages, so normally you should only need to set
+# JSPSYCH_LOCAL_PLUGINS=True above. The variables below can be uncommented and changed to
+# override the host and/or individual ports - only needed if the local lookit-jspsych packages are
+# hosted elsewhere or are served on different ports (e.g. because of a conflict).
+# These should match whatever you're using to serve each package.
+# JSPSYCH_LOCAL_PLUGIN_HOST=http://localhost
+# JSPSYCH_LOCAL_PORT_INITJSPSYCH=10001
+# JSPSYCH_LOCAL_PORT_DATA=10002
+# JSPSYCH_LOCAL_PORT_SURVEYS=10003
+# JSPSYCH_LOCAL_PORT_RECORD=10004
+# JSPSYCH_LOCAL_PORT_STYLE=10005
+# JSPSYCH_LOCAL_PORT_TEMPLATES=10006
+
 # Default repo and branch to use for experiment runner
 EMBER_EXP_PLAYER_BRANCH=master
 EMBER_EXP_PLAYER_REPO=https://github.com/lookit/ember-lookit-frameplayer

exp/tests/test_response_views.py

@@ -2,10 +2,12 @@
 import datetime
 import io
 import json
+import os
 import re
+import tempfile
 import uuid
 import zipfile
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 from django.test import Client, TestCase, override_settings
 from django.urls import reverse
@@ -18,6 +20,7 @@
 from exp.views.responses import (
     StudyResponseSetResearcherFields,
     get_frame_data,
+    write_overview_to_temp_file,
 )
 from exp.views.responses_data import RESPONSE_COLUMNS
 from studies.models import ConsentRuling, Lab, Response, Study, StudyType, Video
@@ -1578,6 +1581,151 @@ def test_psychds_download_excludes_unconsented_responses(self):
                     f"Data from unconsented response found in {name}",
                 )
 
+    def test_write_overview_to_temp_file_handles_non_ascii(self):
+        """Regression test for sentry error LOOKIT-BACKEND-GN.
+
+        The overview temp file must be written as UTF-8 so non-ASCII data
+        (e.g. the acute accent '\xb4') does not raise UnicodeEncodeError under
+        an ASCII default locale. The production uWSGI container defaults to an
+        ASCII locale, so we force that here to reproduce the crash regardless
+        of the host's locale (which is often UTF-8 in development).
+        """
+        header_list = ["response__id", "child__name"]
+        session_list = [{"response__id": "1", "child__name": "Rene\xb4 O’Brien"}]
+
+        # CPython resolves a text-mode file's default encoding from the C
+        # locale, which can't be patched in-process. Instead, simulate the
+        # production container (whose default text encoding is ASCII) by
+        # supplying an ASCII default only when the code under test opens a
+        # text-mode temp file without an explicit encoding. The fix passes
+        # encoding="utf-8", so it is unaffected; the unfixed code is not.
+        real_ntf = tempfile.NamedTemporaryFile
+
+        def ascii_default_ntf(*args, **kwargs):
+            if "b" not in kwargs.get("mode", "r"):
+                kwargs.setdefault("encoding", "ascii")
+            return real_ntf(*args, **kwargs)
+
+        with patch(
+            "exp.views.responses.tempfile.NamedTemporaryFile",
+            side_effect=ascii_default_ntf,
+        ):
+            path = write_overview_to_temp_file(session_list, header_list)
+        try:
+            with open(path, encoding="utf-8") as f:
+                contents = f.read()
+        finally:
+            os.unlink(path)
+        self.assertIn("Rene\xb4 O’Brien", contents)
+
+    def test_psychds_download_with_non_ascii_response_data(self):
+        """The psychds download must not crash on non-ASCII response data."""
+        non_ascii_name = "Rene\xb4"
+        child = G(
+            Child,
+            user=self.non_preview_participant,
+            given_name=non_ascii_name,
+            birthday=datetime.date.today() - datetime.timedelta(366),
+        )
+        response = G(
+            Response,
+            child=child,
+            study=self.study,
+            study_type=self.study.study_type,
+            completed=True,
+            completed_consent_frame=True,
+            sequence=["0-video-config", "1-video-setup", "2-my-consent-frame"],
+            exp_data={
+                "0-video-config": {"frameType": "DEFAULT"},
+                "2-my-consent-frame": {"frameType": "CONSENT"},
+                "3-my-exit-frame": {"frameType": "EXIT", "feedback": non_ascii_name},
+            },
+            demographic_snapshot=self.non_preview_demo,
+        )
+        G(
+            ConsentRuling,
+            response=response,
+            action="accepted",
+            arbiter=self.study_reader,
+        )
+
+        self.client.force_login(self.study_reader)
+        http_response, zip_bytes = self._get_psychds_zip()
+        self.assertEqual(http_response.status_code, 200)
+        # zip is valid and the non-ASCII value round-trips as UTF-8
+        with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
+            contents = b"".join(zf.read(name) for name in zf.namelist())
+        self.assertIn(non_ascii_name.encode("utf-8"), contents)
+
+    def test_framedata_dict_csv_task_handles_non_ascii(self):
+        """Regression test for the frame data dictionary download.
+
+        studies.tasks.build_framedata_dict writes a CSV whose frame IDs and
+        keys come from researcher-authored protocol data, which can be
+        non-ASCII. The temp file must be written as UTF-8 so it does not raise
+        UnicodeEncodeError under the production container's ASCII locale.
+        """
+        from studies.tasks import build_framedata_dict
+
+        # exp_data yields a non-ASCII frame id ("café-frame") and key ("réponse")
+        response = G(
+            Response,
+            child=self.non_preview_child,
+            study=self.study,
+            study_type=self.study.study_type,
+            completed=True,
+            completed_consent_frame=True,
+            sequence=["0-video-config", "1-café-frame"],
+            exp_data={
+                "0-video-config": {"frameType": "DEFAULT"},
+                "1-café-frame": {"frameType": "DEFAULT", "réponse": "oui"},
+            },
+            demographic_snapshot=self.non_preview_demo,
+        )
+        G(
+            ConsentRuling,
+            response=response,
+            action="accepted",
+            arbiter=self.study_reader,
+        )
+
+        # Simulate the container's ASCII default encoding: apply it only when
+        # a text-mode file is opened without an explicit encoding. The fix
+        # passes encoding="utf-8", so it is unaffected; the unfixed code is not.
+        real_open = open
+
+        def ascii_default_open(*args, **kwargs):
+            mode = kwargs.get("mode") or (args[1] if len(args) > 1 else "r")
+            if "b" not in mode:
+                kwargs.setdefault("encoding", "ascii")
+            return real_open(*args, **kwargs)
+
+        uploaded = {}
+
+        def capture_upload(path):
+            with real_open(path, encoding="utf-8") as f:
+                uploaded["contents"] = f.read()
+
+        mock_blob = MagicMock()
+        mock_blob.exists.return_value = False
+        mock_blob.upload_from_filename.side_effect = capture_upload
+        mock_blob.generate_signed_url.return_value = "https://example.com/signed"
+
+        with (
+            override_settings(
+                GS_PROJECT_ID="test-project", GS_PRIVATE_BUCKET_NAME="test-bucket"
+            ),
+            patch("studies.tasks.gc_storage") as mock_gc,
+            patch("studies.tasks.send_mail"),
+            patch("builtins.open", side_effect=ascii_default_open),
+        ):
+            mock_gc.blob.Blob.return_value = mock_blob
+            build_framedata_dict("frames_dict", self.study.uuid, self.study_reader.uuid)
+
+        mock_blob.upload_from_filename.assert_called_once()
+        self.assertIn("café-frame", uploaded["contents"])
+        self.assertIn("réponse", uploaded["contents"])
+
 
 class ResponseViewResearcherUpdateFieldsTestCase(TestCase):
     def setUp(self):

exp/tests/test_runner_views.py

@@ -3,7 +3,7 @@
 from unittest.mock import Mock, patch
 
 import requests
-from django.test import Client, TestCase
+from django.test import Client, TestCase, override_settings
 from django.urls import reverse
 from django_dynamic_fixture import G
 from guardian.shortcuts import assign_perm
@@ -498,3 +498,62 @@ def test_jspsych_preview_context_contains_autoload_plugins(self, mock_aws):
 
         # Should include the autoload plugin
         self.assertIn(self.autoload_plugin, autoload_plugins)
+
+    @override_settings(JSPSYCH_LOCAL_PLUGINS=False)
+    @patch("exp.views.study.get_jspsych_aws_values")
+    def test_preview_chs_plugins_keep_db_urls_without_local_overlay(self, mock_aws):
+        """With JSPSYCH_LOCAL_PLUGINS off, preview CHS plugins use their DB URLs."""
+        mock_aws.return_value = {
+            "accessKeyId": "test-key",
+            "secretAccessKey": "test-secret",
+            "sessionToken": "test-token",
+            "expiration": "2099-12-31T23:59:59Z",
+        }
+        self.client.force_login(self.user)
+        response = self.client.get(
+            reverse(
+                "exp:preview-jspsych",
+                kwargs={"uuid": self.study.uuid, "child_id": self.child.uuid},
+            )
+        )
+
+        chs_plugin = next(
+            p for p in response.context["chs_plugins"] if p.name == "CHS Templates"
+        )
+        self.assertEqual(chs_plugin.url, "https://unpkg.com/@lookit/templates@3.2.0")
+        self.assertEqual(chs_plugin.integrity, "sha384-test3")
+
+    @override_settings(
+        JSPSYCH_LOCAL_PLUGINS=True,
+        JSPSYCH_LOCAL_PLUGIN_URLS={
+            "CHS Templates": "http://localhost:10006/index.browser.js"
+        },
+    )
+    @patch("exp.views.study.get_jspsych_aws_values")
+    def test_preview_chs_plugins_use_local_urls_with_overlay(self, mock_aws):
+        """With JSPSYCH_LOCAL_PLUGINS on, mapped preview CHS plugins point at the local
+        dev server and have their SRI integrity cleared."""
+        mock_aws.return_value = {
+            "accessKeyId": "test-key",
+            "secretAccessKey": "test-secret",
+            "sessionToken": "test-token",
+            "expiration": "2099-12-31T23:59:59Z",
+        }
+        self.client.force_login(self.user)
+        response = self.client.get(
+            reverse(
+                "exp:preview-jspsych",
+                kwargs={"uuid": self.study.uuid, "child_id": self.child.uuid},
+            )
+        )
+
+        chs_plugin = next(
+            p for p in response.context["chs_plugins"] if p.name == "CHS Templates"
+        )
+        self.assertEqual(chs_plugin.url, "http://localhost:10006/index.browser.js")
+        self.assertEqual(chs_plugin.integrity, "")
+        # The overlay only mutates in-memory objects, not the database.
+        self.chs_plugin.refresh_from_db()
+        self.assertEqual(
+            self.chs_plugin.url, "https://unpkg.com/@lookit/templates@3.2.0"
+        )

exp/views/responses.py

@@ -399,7 +399,9 @@ def make_chunk(paginator, page_num, header_options):
 
 
 def write_overview_to_temp_file(session_list, header_list):
-    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv", mode="w")
+    tmp = tempfile.NamedTemporaryFile(
+        delete=False, suffix=".csv", mode="w", encoding="utf-8"
+    )
     tmp.write(",".join(header_list))
     for session_row in session_list:
         tmp.write(
@@ -1762,7 +1764,7 @@ def render_to_response(self, context, **response_kwargs):
         tmp_all_response = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
 
         try:
-            with open(tmp_all_response.name, "w") as f:
+            with open(tmp_all_response.name, "w", encoding="utf-8") as f:
                 for page_num in paginator.page_range:
                     f.write(make_chunk(paginator, page_num, header_options))
 

exp/views/study.py

@@ -4,6 +4,7 @@
 from functools import reduce
 from typing import Any, Dict, NamedTuple, Text
 
+from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.db.models import Q
@@ -27,7 +28,6 @@
     ResearcherLoginRequiredMixin,
     SingleObjectFetchProtocol,
 )
-from project import settings
 from studies.forms import (
     DEFAULT_GENERATOR,
     EFPForm,
@@ -57,6 +57,7 @@
     TRANSITION_LABELS,
 )
 from web.views import (
+    _apply_local_plugin_overlay,
     create_external_response,
     get_external_url,
     get_jspsych_aws_values,
@@ -977,9 +978,14 @@ def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
         context["jspsych_library"] = JSPsychPlugin.objects.filter(
             category=JSPsychPlugin.Category.JSPSYCH_LIBRARY, autoload=True
         ).order_by("order")
-        context["chs_plugins"] = JSPsychPlugin.objects.filter(
+        chs_plugins = JSPsychPlugin.objects.filter(
             category=JSPsychPlugin.Category.CHS_JSPSYCH, autoload=True
         ).order_by("order")
+        # In local development, serve the CHS (@lookit/*) packages from local dev builds
+        # instead of the published URLs stored in the database. No-op in staging/prod.
+        if settings.JSPSYCH_LOCAL_PLUGINS:
+            chs_plugins = _apply_local_plugin_overlay(chs_plugins)
+        context["chs_plugins"] = chs_plugins
         context["autoload_plugins"] = (
             JSPsychPlugin.objects.filter(autoload=True)
             .exclude(