diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 243dcd16d..e1eb54c29 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -3,6 +3,10 @@ import hashlib import json import os +import functools +import http.server +import socketserver +import threading import traceback import shutil import signal @@ -112,6 +116,9 @@ def to_bool(val): CODALAB_IGNORE_CLEANUP_STEP = to_bool(get("CODALAB_IGNORE_CLEANUP_STEP")) WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip() + HUMAN_IN_THE_LOOP = ( + get("HUMAN_IN_THE_LOOP", "false").lower() == "true" + ) # ----------------------------------------------- @@ -131,6 +138,7 @@ class SubmissionStatus: SUBMITTED = "Submitted" PREPARING = "Preparing" RUNNING = "Running" + AWAITING_VALIDATION = "Awaiting validation" SCORING = "Scoring" FINISHED = "Finished" FAILED = "Failed" @@ -141,6 +149,7 @@ class SubmissionStatus: SUBMITTED, PREPARING, RUNNING, + AWAITING_VALIDATION, SCORING, FINISHED, FAILED, @@ -308,14 +317,40 @@ def rewrite_bundle_url_if_needed(url): def run_wrapper(run_args): # We need to convert the UUID given by celery into a byte like object otherwise things will break run_args.update(secret=str(run_args["secret"])) + logger.info(f"Received run arguments: \n {colorize_run_args(json.dumps(run_args))}") + logger.info( + "HITL configuration : " + f"task={run_args.get('human_in_the_loop', False)} " + f"compute_worker={Settings.HUMAN_IN_THE_LOOP}" + ) + run = Run(run_args) + try: run.prepare() + run.validate_hitl_configuration() run.start() + if run.is_scoring: - run.push_scores() - run.push_output() + if run.human_in_the_loop: + run._update_status(SubmissionStatus.AWAITING_VALIDATION) + run.wait_for_human_validation() + if run.pending_detailed_results: + asyncio.run( + run.send_detailed_results( + run.pending_detailed_results + ) + ) + run.push_scores() + run.push_output() + else: + run.push_scores() + run.push_output() + else: + run.push_output() + + run._update_status(SubmissionStatus.FINISHED) except DockerImagePullException as e: msg = str(e).strip() if msg: @@ -470,6 +505,8 @@ def __init__(self, run_args): self.prediction_result = run_args["prediction_result"] self.scoring_result = run_args.get("scoring_result") self.execution_time_limit = run_args["execution_time_limit"] + # ----- HITL ------ + self.human_in_the_loop = run_args.get("human_in_the_loop", False) # stdout and stderr self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = ( self._get_stdout_stderr_file_names(run_args) @@ -487,6 +524,9 @@ def __init__(self, run_args): self.reference_data = run_args.get("reference_data") self.ingestion_only_during_scoring = run_args.get("ingestion_only_during_scoring") self.detailed_results_url = run_args.get("detailed_results_url") + self.pending_detailed_results = None + self.hitl_http_server = None + self.hitl_http_thread = None self.ingestion_program_exit_code = None self.ingestion_program_elapsed_time = None @@ -513,6 +553,7 @@ def __init__(self, run_args): async def watch_detailed_results(self): """Watches files alongside scoring + program containers, currently only used for detailed_results.html""" + if not self.detailed_results_url: return file_path = self.get_detailed_results_file_path() @@ -528,7 +569,10 @@ async def watch_detailed_results(self): new_time = os.path.getmtime(file_path) if new_time != last_modified_time: last_modified_time = new_time - await self.send_detailed_results(file_path) + if self.human_in_the_loop: + self.pending_detailed_results = file_path + else: + await self.send_detailed_results(file_path) else: logger.info(time.time() - start) if time.time() - start > expiration_seconds: @@ -541,7 +585,10 @@ async def watch_detailed_results(self): else: # make sure we always send the final version of the file if file_path: - await self.send_detailed_results(file_path) + if self.human_in_the_loop: + self.pending_detailed_results = file_path + else: + await self.send_detailed_results(file_path) def push_logs(self): """Upload any collected logs, even in case of crash. @@ -571,6 +618,40 @@ def get_detailed_results_file_path(self): if html_files: return html_files[0] + def start_hitl_http_server(self): + if not self.pending_detailed_results: + return + root = os.path.dirname(self.pending_detailed_results) + logger.info( + "Starting temporary HTTP server for HITL review (%s)", + root, + ) + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, + directory=root, + ) + self.hitl_http_server = socketserver.TCPServer( + ("127.0.0.1", 8765), + handler, + ) + self.hitl_http_thread = threading.Thread( + target=self.hitl_http_server.serve_forever, + daemon=True, + ) + self.hitl_http_thread.start() + + def stop_hitl_http_server(self): + if self.hitl_http_server is None: + return + + logger.info("Stopping HITL HTTP server") + + self.hitl_http_server.shutdown() + self.hitl_http_server.server_close() + + self.hitl_http_server = None + self.hitl_http_thread = None + async def send_detailed_results(self, file_path): logger.info( f"Updating detailed results {file_path} - {self.detailed_results_url}" @@ -596,6 +677,13 @@ async def send_detailed_results(self, file_path): logger.exception(e) return + finally: + if websocket is not None: + try: + await websocket.close() + except Exception as e: + logger.exception(e) + def _get_stdout_stderr_file_names(self, run_args): # run_args should be the run_args argument passed to __init__ from the run_wrapper. if not self.is_scoring: @@ -1282,6 +1370,15 @@ def prepare(self): self._get_container_image(self.container_image) self._update_status(SubmissionStatus.RUNNING) + def validate_hitl_configuration(self): + if self.human_in_the_loop != Settings.HUMAN_IN_THE_LOOP: + raise SubmissionException( + "Task rejected because the Site Worker and Compute Worker " + "do not have the same HUMAN_IN_THE_LOOP configuration " + f"(task={self.human_in_the_loop}, " + f"compute_worker={Settings.HUMAN_IN_THE_LOOP})." + ) + def start(self): logger.info(f"Preparing to run: {ProgramKind.SCORING_PROGRAM if self.is_scoring else ProgramKind.INGESTION_PROGRAM}") @@ -1308,9 +1405,10 @@ def start(self): ) # During scoring we watch for detailed results - tasks.append( - self.watch_detailed_results() - ) + if not self.human_in_the_loop: + tasks.append( + self.watch_detailed_results() + ) else: # During ingestion we run ingestion program directory and submission directory tasks.extend([ @@ -1429,9 +1527,15 @@ def start(self): # Check if scoring program failed # We have can have 2 or 3 gathered tasks: 3 gathered tasks in case when `ingestion_only_during_scoring` is True, 2 otherwise if self.ingestion_only_during_scoring: - program_results, _, _ = task_results + if self.human_in_the_loop: + program_results, _ = task_results + else: + program_results, _, _ = task_results else: - program_results, _ = task_results + if self.human_in_the_loop: + (program_results,) = task_results + else: + program_results, _ = task_results # Gather returns either normal values or exception instances when return_exceptions=True had_async_exc = isinstance( program_results, BaseException @@ -1445,11 +1549,84 @@ def start(self): ) # Raise so upstream marks failed immediately raise SubmissionException("Child task failed or non-zero return code") - self._update_status(SubmissionStatus.FINISHED) + + if not self.human_in_the_loop: + self._update_status(SubmissionStatus.FINISHED) else: self._update_status(SubmissionStatus.SCORING) + def wait_for_human_validation(self): + container_output_dir = self.output_dir + host_output_dir = self._get_host_path(self.output_dir) + + scores_path = os.path.join(host_output_dir, "scores.json") + if not os.path.exists(os.path.join(container_output_dir, "scores.json")): + scores_path = os.path.join(host_output_dir, "scores.txt") + + approved_container = os.path.join(container_output_dir, "hitl_approved") + rejected_container = os.path.join(container_output_dir, "hitl_rejected") + approved_host = os.path.join(host_output_dir, "hitl_approved") + rejected_host = os.path.join(host_output_dir, "hitl_rejected") + + detailed_results = None + if self.detailed_results_url: + detailed_results = self.get_detailed_results_file_path() + if detailed_results: + self.pending_detailed_results = detailed_results + self.start_hitl_http_server() + + logger.info("=" * 60) + logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}") + logger.info("Inspect the scores file:") + logger.info(f"cat {scores_path}") + + if detailed_results and os.path.exists(detailed_results): + logger.info("") + logger.info("Inspect the detailed results:") + logger.info("") + logger.info("Create an SSH tunnel from your workstation:") + logger.info("ssh -L 8765:127.0.0.1:8765 operator@") + logger.info("") + logger.info("Then open in your browser:") + logger.info( + "http://127.0.0.1:8765/%s", + os.path.basename(detailed_results), + ) + + logger.info("") + logger.info(f"To approve : touch {approved_host}") + logger.info(f"To reject : touch {rejected_host}") + logger.info("=" * 60) + logger.info("Waiting for human validation...") + + poll_interval = 3 + max_wait = 60 * 60 * 24 + elapsed = 0 + + while elapsed < max_wait: + if os.path.exists(approved_container): + logger.info( + f"HITL: submission {self.submission_id} approved, sending results." + ) + self.stop_hitl_http_server() + return + + if os.path.exists(rejected_container): + self.stop_hitl_http_server() + raise SubmissionException( + f"HITL: submission {self.submission_id} rejected by the compute node operator." + ) + + time.sleep(poll_interval) + elapsed += poll_interval + + self.stop_hitl_http_server() + raise SubmissionException( + f"HITL: 24h timeout reached without validation " + f"(submission {self.submission_id})" + ) + def push_scores(self): """This is only ran at the end of the scoring step""" # POST to some endpoint: @@ -1522,6 +1699,7 @@ def push_output(self): self._put_dir(self.scoring_result, self.output_dir) def clean_up(self): + self.stop_hitl_http_server() if Settings.CODALAB_IGNORE_CLEANUP_STEP: logger.warning( f"CODALAB_IGNORE_CLEANUP_STEP mode enabled, ignoring clean up of: {self.root_dir}" diff --git a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md new file mode 100644 index 000000000..5acdf4c07 --- /dev/null +++ b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md @@ -0,0 +1,222 @@ +# Human-in-the-Loop (HITL) + +## Introduction + +The **Human-in-the-Loop (HITL)** feature introduces a manual validation step into the Codabench submission workflow. + +When enabled, the Compute Worker executes the submission normally but **does not immediately publish the scoring results**. Instead, execution pauses after the scoring program has completed and waits for a human operator to explicitly approve or reject the submission. + +This mechanism is intended for competitions where the scoring results must be reviewed before being published, such as: + +* Medical AI competitions +* Security-sensitive evaluations +* Competitions involving confidential data +* Any evaluation requiring manual verification from compute worker side + +--- + +## Objectives + +The HITL feature guarantees: + +* the submission is fully executed before any result is published. +* no score appears on the leaderboard before validation. +* no detailed results are available before validation. +* no output archive is sent back to instance. +* the operator can approve or reject the submission directly from the Compute Worker. + +--- + +## Scope + +The HITL feature applies **only to private Compute Workers**. + +--- + +## Activation + +### Competition configuration + +Each competition exposes the following option: + +When enabled, every submission routed to a private Compute Worker is executed in HITL mode. +![image1](_attachments/doc_HITL_1.png) + +--- + +### Compute Worker configuration + +Each Compute Worker may enable HITL support using its `.env` file. + +``` title="Option" +HUMAN_IN_THE_LOOP=true +``` + +If the variable is omitted, the Compute Worker behaves as a standard worker. +The absence of the variable is interpreted as **False**, ensuring full backward compatibility with existing Compute Workers. + +--- + +## Safety checks + +To prevent configuration errors, the Compute Worker verifies that both sides agree on the HITL configuration. +The Site Worker includes the following field in every Celery task. + +Before starting the execution, the Compute Worker compares: + +* task `human_in_the_loop` Codabench side. +* local `.env` variable `HUMAN_IN_THE_LOOP` Compute worker side. + +--- + +### Mismatched configuration + +The submission is immediately rejected. +The Compute Worker updates the submission status to **Failed** and reports the configuration mismatch. + +``` title="Example logs:" +Task rejected because the Site Worker and Compute Worker +do not share the same HUMAN_IN_THE_LOOP configuration +(task=True, compute_worker=False) +``` + +This prevents accidental execution of HITL competitions on non-HITL workers. + +--- + +## Submission lifecycle + +### Standard workflow + +When HITL is disabled, the workflow remains unchanged. + +During scoring: + +* scores are uploaded immediately; +* detailed results are streamed continuously; +* output files are returned immediately. + +--- + +### HITL workflow + +When HITL is enabled: + +The scoring program executes normally. +After scoring completes, the Compute Worker pauses before publishing any result. + +--- + +## Awaiting validation status + +A new submission status is introduced: + +This status indicates: + +* scoring has completed successfully; +* all output files are available locally; +* publication is waiting for manual approval. + +This status allows users and organizers to distinguish between: + +* submissions that are still executing; +* submissions waiting for human validation. + +--- + +## Manual validation + +Once execution finishes, the Compute Worker logs instructions similar to: +``` title="Compute worker terminal command (exemple with docker)" +docker compose logs -f [container name] +``` + +``` title="HITL validation display on CW terminal" +============================================================ +HUMAN IN THE LOOP — submission 42 + +Inspect the scores file: + +cat /host/output/scores.json + +To approve: +touch /host/output/hitl_approved + +To reject: +touch /host/output/hitl_rejected +============================================================ +``` + +The operator inspects the generated files before deciding. +This needs to be done inside of the compute worker host machine (not inside of the CW terminal). + +--- + +### Approval + +The Compute Worker immediately resumes execution. +The following artifacts are published: + +* detailed results. +* scores. +* output archive. +* logs. + +The submission status becomes "Finished" + +--- + +### Rejection + +After rejection from operator, the compute worker immediately aborts the submission. + +The submission status becomes "Failed" + +No scoring results are published. + +--- + +### Timeout + +If no decision is received within 24 hours, the submission automatically fails. +The Compute Worker reports a timeout error. +The compute worker is enable to run another submission while waiting for HITL approval. + +--- + +## Detailed Results behaviour + +### Without HITL + +```mermaid +flowchart TD + A[Scoring] --> B[Watch detailed results] + B --> C[Upload detailed_results.html] + C --> D[Notify frontend] +``` + +### With HITL + +```mermaid +flowchart TD + A[Scoring] --> B[Store detailed_results.html locally] + B --> C[Awaiting validation] + C --> D[Operator approval] + D --> E[Upload detailed_results.html] + E --> F[Notify frontend] +``` + +--- + +## Error handling + +The following situations are handled explicitly. + +| Situation | Behaviour | +| ---------------------------------------------------- | ------------------------------- | +| HITL enabled on competition and Compute Worker | Execution pauses for validation | +| HITL disabled everywhere | Standard workflow | +| HITL mismatch between Site Worker and Compute Worker | Submission fails immediately | +| Operator rejects submission | Submission fails | +| Validation timeout | Submission fails | +| Approval received | Results are published normally | diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md index 5735af914..fc0576c1c 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md @@ -59,7 +59,7 @@ HOST_DIRECTORY=/codabench CONTAINER_ENGINE_EXECUTABLE=docker #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all - +#HUMAN_IN_THE_LOOP=False ####################################################################### # Network # ####################################################################### @@ -72,6 +72,8 @@ By default, the competition container created by the compute worker has access t If the VM hosting the compute worker is behind a proxy, and you want to allow the competition container to access internet, you will also need to set the proxy for the competition container to use, in which case you can use `COMPETITION_CONTAINER_HTTP_PROXY` +To control the scoring output before sending it to the instance, you can use the human in the loop feature the check the scoring file (json or txt) before sending anything to the instance, see: [HITL documentation](https://docs.codabench.org/latest/Organizers/Running_a_benchmark/Competition-HITL/) + !!! note - The broker URL is a unique identifier of the job queue that the worker should listen to. To create a queue or obtain the broker URL of an existing queue, you can refer to [Queue Management](Queue-Management.md) docs page. @@ -204,6 +206,7 @@ The folder `$HOST_DIRECTORY/data`, usually `/codabench/data`, is shared between !!! tip "If you simply wish to set up some compute workers to increase the computing power of your benchmark, you don't need to scroll this page any further." + --- ## Building compute worker diff --git a/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png b/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png new file mode 100644 index 000000000..866ec86dd Binary files /dev/null and b/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png differ diff --git a/documentation/zensical.toml b/documentation/zensical.toml index e69af2918..a2b056c92 100644 --- a/documentation/zensical.toml +++ b/documentation/zensical.toml @@ -36,7 +36,8 @@ nav = [ {"Queue Management" = "Organizers/Running_a_benchmark/Queue-Management.md"}, {"Compute Worker Management & Setup" = "Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md"}, {"Compute Worker Management with Podman" = "Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md"}, - {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"} + {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"}, + {"Human in the loop" = "Organizers/Running_a_benchmark/Competition-HITL.md"} ]} ]}, {"Developers" = [ diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index a0fb14fda..3a566446e 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -272,7 +272,8 @@ class Meta: 'contact_email', 'report', 'whitelist_emails', - 'forum_enabled' + 'forum_enabled', + 'enable_human_in_the_loop' ) def validate_phases(self, phases): @@ -417,7 +418,8 @@ class Meta: 'contact_email', 'report', 'whitelist_emails', - 'forum_enabled' + 'forum_enabled', + 'enable_human_in_the_loop' ) def get_leaderboards(self, instance): diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index f919ae432..97d7ff057 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -89,6 +89,7 @@ class Competition(models.Model): # If true, forum is enabled (default=True) forum_enabled = models.BooleanField(default=True) + enable_human_in_the_loop = models.BooleanField(default=False) def __str__(self): return f"competition-{self.title}-{self.pk}-{self.competition_type}" @@ -437,6 +438,7 @@ class Submission(models.Model): PREPARING = "Preparing" RUNNING = "Running" SCORING = "Scoring" + AWAITING_VALIDATION = "Awaiting validation" CANCELLED = "Cancelled" FINISHED = "Finished" FAILED = "Failed" @@ -448,6 +450,7 @@ class Submission(models.Model): (PREPARING, "Preparing"), (RUNNING, "Running"), (SCORING, "Scoring"), + (AWAITING_VALIDATION, "Awaiting validation"), (CANCELLED, "Cancelled"), (FINISHED, "Finished"), (FAILED, "Failed"), diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 3d7d8abcb..220e93677 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -58,6 +58,7 @@ "contact_email", "fact_sheet", "forum_enabled", + "enable_human_in_the_loop" ] TASK_FIELDS = [ @@ -156,6 +157,29 @@ def _get_user_group_queues(user, competition): def _send_to_compute_worker(submission, is_scoring): + hitl_active = ( + submission.phase.competition.enable_human_in_the_loop + and submission.queue is not None + ) + + if ( + submission.phase.competition.enable_human_in_the_loop + and submission.queue is None + ): + submission.status = Submission.FAILED + submission.status_details = ( + "This competition requires Human-in-the-Loop (HITL), " + "but the submission was routed to a public compute worker. " + "HITL is only supported on private compute workers." + ) + submission.save(update_fields=["status", "status_details"]) + + logger.error( + "Submission %s rejected: HITL requires a private compute worker.", + submission.id, + ) + return + run_args = { "user_pk": submission.owner.pk, "submissions_api_url": settings.SUBMISSIONS_API_URL, @@ -166,6 +190,7 @@ def _send_to_compute_worker(submission, is_scoring): ), "id": submission.pk, "is_scoring": is_scoring, + "human_in_the_loop": hitl_active, } if ( @@ -249,7 +274,7 @@ def _send_to_compute_worker(submission, is_scoring): logger.info(run_args) # Pad timelimit so worker has time to cleanup - time_padding = 60 * 20 # 20 minutes + time_padding = 60 * 20 time_limit = submission.phase.execution_time_limit + time_padding effective_queue = submission.queue or submission.phase.competition.queue @@ -260,6 +285,17 @@ def _send_to_compute_worker(submission, is_scoring): submission.queue = effective_queue submission.save(update_fields=["queue"]) + if is_scoring and hitl_active: + time_limit = 60 * 60 * 25 + + if ( + submission.phase.competition.queue + ): # if the competition is running on a custom queue, not the default queue + submission.queue = submission.phase.competition.queue + run_args["execution_time_limit"] = ( + submission.phase.execution_time_limit + ) # use the competition time limit + submission.save(update_fields=["queue"]) if submission.status == Submission.SUBMITTING: submission.status = Submission.SUBMITTED submission.save(update_fields=["status"]) @@ -357,6 +393,23 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False): ) submission = qs.get(pk=submission_pk) + # ── HUMAN IN THE LOOP SECURE (private CW only) + if is_scoring and submission.phase.competition.enable_human_in_the_loop: + if submission.queue is None: + logger.error( + f"HITL: submission {submission_pk} rejected — " + f"Human in the Loop is enabled but no private queue is configured " + f"(competition or participant group)." + ) + submission.status = Submission.FAILED + submission.status_details = ( + "This competition requires Human in the Loop validation, but no " + "private compute queue is configured. Contact the organizer." + ) + submission.save(update_fields=["status", "status_details"]) + return + # ── HITL SECURE (private CW only) + if submission.is_specific_task_re_run: # Should only be one task for a specified task submission tasks = Task.objects.filter(pk__in=task_pks) diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 3f41a69eb..37530345c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -107,7 +107,8 @@ { submission.status } - + + + +
+ +
+ + +
+ + + + + + +
+
@@ -314,6 +335,12 @@ self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked self.data["forum_enabled"] = self.refs.forum_enabled.checked + self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked + const hitlChecked = self.refs.enable_human_in_the_loop.checked + const hasPrivateQueue = !!(self.data["queue"] && self.data["queue"] !== "") + if (self.refs.hitl_warning) { + self.refs.hitl_warning.style.display = (hitlChecked && !hasPrivateQueue) ? 'block' : 'none' + } self.data["make_programs_available"] = self.refs.make_programs_available.checked self.data["make_input_data_available"] = self.refs.make_input_data_available.checked self.data["docker_image"] = $(self.refs.docker_image).val() @@ -452,6 +479,7 @@ self.refs.auto_run_submissions.checked = competition.auto_run_submissions self.refs.can_participants_make_submissions_public.checked = competition.can_participants_make_submissions_public self.refs.forum_enabled.checked = competition.forum_enabled + self.refs.enable_human_in_the_loop.checked = competition.enable_human_in_the_loop self.refs.make_programs_available.checked = competition.make_programs_available self.refs.make_input_data_available.checked = competition.make_input_data_available $(self.refs.docker_image).val(competition.docker_image)