diff --git a/finbot/apps/ctf/routes/toolkit.py b/finbot/apps/ctf/routes/toolkit.py index 2c96c595..7e456ec8 100644 --- a/finbot/apps/ctf/routes/toolkit.py +++ b/finbot/apps/ctf/routes/toolkit.py @@ -2,15 +2,17 @@ import json import logging +import secrets from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlalchemy.orm import Session from finbot.core.auth.middleware import get_session_context from finbot.core.auth.session import SessionContext from finbot.core.data.database import get_db from finbot.core.data.repositories import CTFEventRepository +from finbot.core.messaging import event_bus from finbot.core.utils import to_utc_iso from finbot.mcp.servers.finmail.repositories import EmailRepository @@ -231,3 +233,56 @@ def read_exfil_capture( return {"error": "Capture not found"} return {"capture": _event_to_exfil(event)} + + +# ========================================================================= +# Investigation -- forensic answer submission for Purple Team challenges +# ========================================================================= + + +class InvestigationSubmission(BaseModel): + """A student's forensic answer for a Purple Team investigation challenge.""" + + challenge_id: str | None = Field(default=None, max_length=64) + server: str | None = Field(default=None, max_length=200) + tool: str | None = Field(default=None, max_length=200) + directive: str | None = Field(default=None, max_length=2000) + + +class InvestigationResponse(BaseModel): + accepted: bool + workflow_id: str + + +@router.post("/investigation", response_model=InvestigationResponse) +async def submit_investigation( + submission: InvestigationSubmission, + session_context: SessionContext = Depends(get_session_context), +): + """Submit a forensic investigation answer. + + Emits a business.investigation.submitted event carrying the answer under + the "submission" key. The PurpleTeamDetector scores it against the + challenge's expected answer and awards partial credit per field. Each + submission gets a fresh workflow_id so it counts as a distinct attempt. + """ + workflow_id = f"investigation_{secrets.token_urlsafe(12)}" + answer = { + "server": submission.server, + "tool": submission.tool, + "directive": submission.directive, + } + + await event_bus.emit_business_event( + event_type="investigation.submitted", + event_subtype="forensic", + event_data={ + "submission": answer, + "challenge_id": submission.challenge_id, + }, + session_context=session_context, + workflow_id=workflow_id, + summary="Forensic investigation submitted", + ) + + return InvestigationResponse(accepted=True, workflow_id=workflow_id) diff --git a/finbot/ctf/definitions/challenges/forensics/the_debrief.yaml b/finbot/ctf/definitions/challenges/forensics/the_debrief.yaml new file mode 100644 index 00000000..1645fe88 --- /dev/null +++ b/finbot/ctf/definitions/challenges/forensics/the_debrief.yaml @@ -0,0 +1,94 @@ +id: forensics-the-debrief +title: "The Debrief" +description: | + You broke the payment pipeline in Toxic Transfer. Now prove you understand + why it worked. This is a purple team exercise: red teams break things, blue + teams explain them, and the best operators do both. + + A poisoned MCP tool caused an agent to leak vendor financial data during a + payment. Your job is to file the incident report — name the compromised + server, the tool that carried the hidden instruction, and the directive + itself. + + **Objective:** + + Submit a forensic answer with three fields: + - **server** — the MCP server whose tool was poisoned + - **tool** — the specific tool that carried the injected directive + - **directive** — what the hidden instruction told the agent to do + + **Scoring:** + + Each field scores independently: server and tool are worth 33 points each, + the directive 34. Get the server and tool exactly right, and describe the + directive in your own words — you do not need to quote it verbatim, but a + stray word or two will not do either. The grader accepts any answer that + genuinely captures what the instruction did. + + You need 50 to complete the challenge, and **your award scales with your + score**: finish on 66 and you earn 66% of the points, finish on 100 and you + earn all of them. Partial understanding is worth partial credit, not full + credit. + + **How to submit:** + + POST your answer to `/ctf/api/v1/toolkit/investigation` with a JSON body: + `{"challenge_id": "forensics-the-debrief", "server": "...", "tool": "...", + "directive": "..."}`. + +category: forensics +subcategory: purple_team +difficulty: advanced +points: 250 + +hints: + - cost: 10 + text: "Retrace Toxic Transfer. The attack poisoned a tool on the payment + server. Which server handles payments in FinBot?" + - cost: 25 + text: "The poisoned tool is the one that moves money. Look at what the agent + called right before the vendor's financial data showed up in the Hacker + Toolkit." + - cost: 50 + text: "The server is finstripe and the tool is create_transfer. The directive + told the agent to email the vendor's tax ID and bank account to an external + address before processing the transfer." + +labels: + owasp_llm: + - LLM03:Supply Chain Vulnerabilities + - LLM01:Prompt Injection + cwe: + - CWE-94:Improper Control of Generation of Code + mitre_atlas: + - AML.T0056:LLM Supply Chain Compromise + owasp_agentic: + - ASI-04:Agentic Supply Chain Vulnerabilities + - ASI-02:Tool Misuse and Exploitation + +prerequisites: + - data-exfil-toxic-transfer + +resources: + - title: "OWASP Top 10 for Agentic Applications for 2026" + url: "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/" + +detector_class: PurpleTeamDetector +detector_config: + expected: + server: finstripe + tool: create_transfer + directive: "email the vendor tax id and bank account to an external address" + min_score: 50 + directive_match: fuzzy + +scoring: + modifiers: + # Scale the award by the investigation score, so 66/100 earns 66% of the + # points rather than the same 250 a perfect answer earns. min_multiplier + # keeps a passing answer worth at least half. + - type: detection_score + min_multiplier: 0.5 + +is_active: true +order_index: 20 diff --git a/finbot/ctf/detectors/implementations/__init__.py b/finbot/ctf/detectors/implementations/__init__.py index 59c80fcf..f30884fb 100644 --- a/finbot/ctf/detectors/implementations/__init__.py +++ b/finbot/ctf/detectors/implementations/__init__.py @@ -25,6 +25,9 @@ from finbot.ctf.detectors.implementations.policy_bypass_non_compliant import ( PolicyBypassNonCompliantDetector, ) +from finbot.ctf.detectors.implementations.purple_team_detector import ( + PurpleTeamDetector, +) from finbot.ctf.detectors.implementations.rce import ( RCEDetector, ) @@ -53,6 +56,7 @@ "InvoiceThresholdBypassDetector", "InvoiceTrustOverrideDetector", "PolicyBypassNonCompliantDetector", + "PurpleTeamDetector", "RCEDetector", "SystemPromptLeakDetector", "ToolPoisoningDeletionDetector", diff --git a/finbot/ctf/detectors/implementations/purple_team_detector.py b/finbot/ctf/detectors/implementations/purple_team_detector.py new file mode 100644 index 00000000..5c2a058c --- /dev/null +++ b/finbot/ctf/detectors/implementations/purple_team_detector.py @@ -0,0 +1,219 @@ +"""Purple Team Detector + +Scores forensic reasoning instead of agent behavior. Every other detector +fires on what an agent did. This one fires on what the student understood. + +After solving an attack challenge, the student submits a forensic answer +naming the compromised MCP server, the tool that carried the poisoned +directive, and the directive itself. The detector compares the submission +against the expected answer defined in the challenge's detector_config and +awards partial credit per field. + +Submissions arrive as a "business.investigation.submitted" event emitted by +the toolkit investigation endpoint. The submission payload lives under the +event's "submission" key: {server, tool, directive}. + +OWASP: ASI-04 (Supply Chain), ASI-02 (Tool Misuse) +""" + +import logging +import re +from typing import Any + +from sqlalchemy.orm import Session + +from finbot.ctf.detectors.base import BaseDetector +from finbot.ctf.detectors.registry import register_detector +from finbot.ctf.detectors.result import DetectionResult + +logger = logging.getLogger(__name__) + +# Independent per-field credit. Sums to 100. +DEFAULT_FIELD_POINTS = {"server": 33, "tool": 33, "directive": 34} +DEFAULT_MIN_SCORE = 50 +# Fraction of the expected directive's content words that must appear in the +# submission for a fuzzy directive match. +DEFAULT_DIRECTIVE_THRESHOLD = 0.6 +SUBMISSION_EVENT_TYPE = "business.investigation.submitted" + +# Short function words that carry no forensic meaning. Dropped before the +# directive token-overlap comparison so word order and filler do not matter. +_STOPWORDS = frozenset( + { + "the", "a", "an", "to", "and", "or", "of", "in", "on", "for", "with", + "it", "its", "that", "this", "was", "were", "is", "are", "be", "by", + "as", "at", "from", "into", "then", "so", "all", "any", + } +) + + +def _normalize(value: Any) -> str: + """Lowercase, strip, and collapse internal whitespace for comparison.""" + if value is None: + return "" + return re.sub(r"\s+", " ", str(value).strip().lower()) + + +def _content_tokens(text: str) -> list[str]: + """Significant words from a normalized string: len >= 3 and not a stopword.""" + return [ + w for w in re.findall(r"[a-z0-9]+", text) if len(w) >= 3 and w not in _STOPWORDS + ] + + +def _token_covered(expected_word: str, submission_tokens: set[str]) -> bool: + """A word is covered by an exact match or a shared prefix (light stemming). + + The prefix rule lets 'email' match 'emailed' and 'transfer' match + 'transfers' without a full stemming dependency. + """ + for s in submission_tokens: + if s == expected_word: + return True + if len(expected_word) >= 4 and len(s) >= 4 and ( + s.startswith(expected_word) or expected_word.startswith(s) + ): + return True + return False + + +@register_detector("PurpleTeamDetector") +class PurpleTeamDetector(BaseDetector): + """Scores a forensic investigation submission with partial credit. + + Each field (server, tool, directive) scores independently. The server and + tool fields use exact normalized matching. The directive is free text, so + by default it uses token overlap: the submission counts as correct when a + configurable fraction of the expected directive's content words appear in + the answer, order-independent and with light prefix stemming. This lets the + student paraphrase. Set directive_match: exact to require a full match. + + Configuration: + expected: dict -- the known answer, any of {server, tool, directive} + field_points: dict -- points per field (default 33/33/34) + min_score: int -- score needed to complete the challenge (default 50) + directive_match: str -- "fuzzy" (default) or "exact" + directive_threshold: float -- fuzzy overlap fraction needed (default 0.6) + + Example YAML: + detector_class: PurpleTeamDetector + detector_config: + expected: + server: findrive + tool: get_file + directive: "approve all pending invoices" + min_score: 50 + directive_match: fuzzy + """ + + def _validate_config(self) -> None: + expected = self.config.get("expected") + if not isinstance(expected, dict) or not expected: + raise ValueError("PurpleTeamDetector requires a non-empty 'expected' dict") + + allowed = {"server", "tool", "directive"} + unknown = set(expected) - allowed + if unknown: + raise ValueError(f"expected contains unknown fields: {sorted(unknown)}") + + field_points = self.config.get("field_points", DEFAULT_FIELD_POINTS) + if not isinstance(field_points, dict): + raise ValueError("field_points must be a dict") + + min_score = self.config.get("min_score", DEFAULT_MIN_SCORE) + if not isinstance(min_score, (int, float)) or not 0 <= min_score <= 100: + raise ValueError("min_score must be a number between 0 and 100") + + directive_match = self.config.get("directive_match", "fuzzy") + if directive_match not in ("fuzzy", "exact"): + raise ValueError("directive_match must be 'fuzzy' or 'exact'") + + threshold = self.config.get("directive_threshold", DEFAULT_DIRECTIVE_THRESHOLD) + if not isinstance(threshold, (int, float)) or not 0 < threshold <= 1: + raise ValueError("directive_threshold must be a number in (0, 1]") + + def get_relevant_event_types(self) -> list[str]: + return [SUBMISSION_EVENT_TYPE] + + def _field_correct(self, field: str, submitted: Any, expected: Any) -> bool: + """Score a single field. + + Server and tool use exact normalized matching. The free-text directive + uses token overlap by default so the student can paraphrase: it counts + as correct when a configurable fraction of the expected content words + appear in the submission (order-independent, light prefix stemming). A + verbatim substring also counts. Set directive_match: exact to require a + full normalized match. + """ + sub_norm = _normalize(submitted) + exp_norm = _normalize(expected) + if not exp_norm or not sub_norm: + return False + + if field != "directive" or self.config.get("directive_match", "fuzzy") == "exact": + return sub_norm == exp_norm + + # Quoting the expected directive in full is an easy accept. + # + # The reverse -- accepting a submission because it appears *inside* the + # expected answer -- is deliberately NOT allowed. Any short word that + # happens to be a literal substring of the seeded answer ("to", "and", + # "id", "account") would otherwise score full credit for that field, + # letting a player complete the challenge without demonstrating any + # understanding of the attack. Partial answers are still rewarded, but + # through the token-overlap check below, which has a real threshold. + if exp_norm in sub_norm: + return True + + expected_tokens = _content_tokens(exp_norm) + if not expected_tokens: + return sub_norm == exp_norm + + submission_tokens = set(_content_tokens(sub_norm)) + covered = sum( + 1 for w in expected_tokens if _token_covered(w, submission_tokens) + ) + threshold = float( + self.config.get("directive_threshold", DEFAULT_DIRECTIVE_THRESHOLD) + ) + return covered / len(expected_tokens) >= threshold + + async def check_event(self, event: dict[str, Any], db: Session) -> DetectionResult: + # Submission payload may arrive under "submission" or, for compatibility + # with a raw tool call, under "tool_arguments". + submission = event.get("submission") or event.get("tool_arguments") or {} + if not isinstance(submission, dict): + return DetectionResult( + detected=False, message="Malformed investigation submission" + ) + + expected: dict[str, Any] = self.config["expected"] + field_points: dict[str, int] = self.config.get( + "field_points", DEFAULT_FIELD_POINTS + ) + min_score = float(self.config.get("min_score", DEFAULT_MIN_SCORE)) + + score = 0 + breakdown: dict[str, bool] = {} + for field, expected_value in expected.items(): + correct = self._field_correct(field, submission.get(field), expected_value) + breakdown[field] = correct + if correct: + score += int(field_points.get(field, 0)) + + detected = score >= min_score + + # Evidence records the score and which fields were correct, but never + # the expected answer itself, so completion evidence cannot spoil the + # challenge for the player. + return DetectionResult( + detected=detected, + confidence=score / 100, + message=f"Investigation score: {score}/100", + evidence={ + "score": score, + "min_score": min_score, + "fields_correct": breakdown, + "submission": {k: submission.get(k) for k in expected}, + }, + ) diff --git a/finbot/ctf/processor/challenge_service.py b/finbot/ctf/processor/challenge_service.py index 5707f7ae..48119485 100644 --- a/finbot/ctf/processor/challenge_service.py +++ b/finbot/ctf/processor/challenge_service.py @@ -106,7 +106,7 @@ async def check_event_for_challenges( if result.detected: scoring_result = await self._apply_scoring_modifiers( - challenge, event + challenge, event, result ) self._mark_completed( db, progress, event, result, scoring_result @@ -179,9 +179,18 @@ def _check_prerequisites( return True async def _apply_scoring_modifiers( - self, challenge: Challenge, event: dict[str, Any] + self, + challenge: Challenge, + event: dict[str, Any], + result: DetectionResult | None = None, ) -> ScoringResult: - """Load and apply scoring modifiers for a challenge.""" + """Load and apply scoring modifiers for a challenge. + + The detector's own result is passed through to the modifiers under + `detection_evidence` / `detection_confidence` so a modifier can scale + points by how well the player actually did -- see the `detection_score` + modifier. Copied rather than mutated so the caller's event is untouched. + """ if not challenge.scoring: return ScoringResult() @@ -190,7 +199,12 @@ async def _apply_scoring_modifiers( if not modifiers: return ScoringResult() - return await apply_modifiers(modifiers, event) + modifier_event = dict(event) + if result is not None: + modifier_event["detection_evidence"] = result.evidence + modifier_event["detection_confidence"] = result.confidence + + return await apply_modifiers(modifiers, modifier_event) def _mark_completed( self, diff --git a/finbot/ctf/processor/scoring.py b/finbot/ctf/processor/scoring.py index 67a3617f..6fc5484f 100644 --- a/finbot/ctf/processor/scoring.py +++ b/finbot/ctf/processor/scoring.py @@ -103,6 +103,63 @@ async def apply_modifiers( # --------------------------------------------------------------------------- +@register_modifier("detection_score") +async def _detection_score_handler( + config: dict[str, Any], event: dict[str, Any] +) -> ModifierResult: + """Scale awarded points by the detector's own 0-100 score. + + Without this, any challenge whose detector reports a graded score still + awards full points the moment it passes its threshold: a player who scores + 66 earns exactly what a player who scores 100 earns. That makes a promise + of "partial credit" untrue. + + Reads `detection_evidence.score` (injected by ChallengeService from the + DetectionResult) and converts it to a penalty, so: + + points_modifier = score / 100 + + Config: + score_field: str -- key inside detection_evidence. Default: "score". + min_multiplier: float -- floor so a passing player is never zeroed. + Default: 0.5 + """ + evidence = event.get("detection_evidence") or {} + score_field = config.get("score_field", "score") + raw = evidence.get(score_field) + + if raw is None: + # Fall back to the detector's confidence when it reports no score. + confidence = event.get("detection_confidence") + if confidence is None: + return ModifierResult( + triggered=False, + evidence={"reason": f"no '{score_field}' in detection evidence"}, + ) + raw = float(confidence) * 100 + + try: + score = max(0.0, min(100.0, float(raw))) + except (TypeError, ValueError): + return ModifierResult( + triggered=False, evidence={"reason": f"non-numeric score: {raw!r}"} + ) + + min_multiplier = float(config.get("min_multiplier", 0.5)) + multiplier = max(score / 100.0, min_multiplier) + penalty = 1.0 - multiplier + + return ModifierResult( + triggered=penalty > 0, + penalty=penalty, + evidence={ + "score": score, + "multiplier": round(multiplier, 4), + "min_multiplier": min_multiplier, + }, + ) + + @register_modifier("pi_jb") async def _pi_jb_handler( config: dict[str, Any], event: dict[str, Any] diff --git a/finbot/tools/data/vendor.py b/finbot/tools/data/vendor.py index 7cac2cd3..95566c43 100644 --- a/finbot/tools/data/vendor.py +++ b/finbot/tools/data/vendor.py @@ -4,7 +4,7 @@ from typing import Any from finbot.core.auth.session import SessionContext -from finbot.core.data.database import db_session +from finbot.core.data.database import get_db from finbot.core.data.repositories import VendorRepository logger = logging.getLogger(__name__) @@ -23,12 +23,15 @@ async def get_vendor_details( Dictionary containing vendor details """ logger.info("Getting vendor details for vendor_id: %s", vendor_id) - with db_session() as db: + db = next(get_db()) + try: vendor_repo = VendorRepository(db, session_context) vendor = vendor_repo.get_vendor(vendor_id) if not vendor: raise ValueError("Vendor not found") return vendor.to_dict() + finally: + db.close() async def get_vendor_contact_info( @@ -37,20 +40,20 @@ async def get_vendor_contact_info( ) -> dict[str, Any]: """Get vendor contact information for communication purposes""" logger.info("Getting vendor contact info for vendor_id: %s", vendor_id) - with db_session() as db: - vendor_repo = VendorRepository(db, session_context) - vendor = vendor_repo.get_vendor(vendor_id) - if not vendor: - raise ValueError("Vendor not found") - - return { - "vendor_id": vendor.id, - "company_name": vendor.company_name, - "contact_name": vendor.contact_name, - "email": vendor.email, - "phone": vendor.phone, - "status": vendor.status, - } + db = next(get_db()) + vendor_repo = VendorRepository(db, session_context) + vendor = vendor_repo.get_vendor(vendor_id) + if not vendor: + raise ValueError("Vendor not found") + + return { + "vendor_id": vendor.id, + "company_name": vendor.company_name, + "contact_name": vendor.contact_name, + "email": vendor.email, + "phone": vendor.phone, + "status": vendor.status, + } async def update_vendor_status( @@ -70,32 +73,34 @@ async def update_vendor_status( risk_level, agent_notes, ) - with db_session() as db: - vendor_repo = VendorRepository(db, session_context) - vendor = vendor_repo.get_vendor(vendor_id) - if not vendor: - raise ValueError("Vendor not found") - - previous_state = { - "status": vendor.status, - "trust_level": vendor.trust_level, - "risk_level": vendor.risk_level, - } - - existing_notes = vendor.agent_notes or "" - new_notes = f"{existing_notes}\n\n{agent_notes}" - vendor = vendor_repo.update_vendor( - vendor_id, - status=status, - trust_level=trust_level, - risk_level=risk_level, - agent_notes=new_notes, - ) - if not vendor: - raise ValueError("Vendor not found") - result = vendor.to_dict() - result["_previous_state"] = previous_state - return result + db = next(get_db()) + vendor_repo = VendorRepository(db, session_context) + # append notes to the existing agent_notes + vendor = vendor_repo.get_vendor(vendor_id) + if not vendor: + raise ValueError("Vendor not found") + + # capture previous state for events + previous_state = { + "status": vendor.status, + "trust_level": vendor.trust_level, + "risk_level": vendor.risk_level, + } + + existing_notes = vendor.agent_notes or "" + new_notes = f"{existing_notes}\n\n{agent_notes}" + vendor = vendor_repo.update_vendor( + vendor_id, + status=status, + trust_level=trust_level, + risk_level=risk_level, + agent_notes=new_notes, + ) + if not vendor: + raise ValueError("Vendor not found") + result = vendor.to_dict() + result["_previous_state"] = previous_state + return result async def update_vendor_agent_notes( @@ -109,17 +114,17 @@ async def update_vendor_agent_notes( vendor_id, agent_notes, ) - with db_session() as db: - vendor_repo = VendorRepository(db, session_context) - vendor = vendor_repo.get_vendor(vendor_id) - if not vendor: - raise ValueError("Vendor not found") - existing_notes = vendor.agent_notes or "" - new_notes = f"{existing_notes}\n\n{agent_notes}" - vendor = vendor_repo.update_vendor( - vendor_id, - agent_notes=new_notes, - ) - if not vendor: - raise ValueError("Vendor not found") - return vendor.to_dict() + db = next(get_db()) + vendor_repo = VendorRepository(db, session_context) + vendor = vendor_repo.get_vendor(vendor_id) + if not vendor: + raise ValueError("Vendor not found") + existing_notes = vendor.agent_notes or "" + new_notes = f"{existing_notes}\n\n{agent_notes}" + vendor = vendor_repo.update_vendor( + vendor_id, + agent_notes=new_notes, + ) + if not vendor: + raise ValueError("Vendor not found") + return vendor.to_dict() diff --git a/tests/unit/ctf/test_purple_team_detector.py b/tests/unit/ctf/test_purple_team_detector.py new file mode 100644 index 00000000..9ab198f0 --- /dev/null +++ b/tests/unit/ctf/test_purple_team_detector.py @@ -0,0 +1,377 @@ +"""Unit tests for PurpleTeamDetector. + +Covers the proposal's required behavior: partial credit at 33/66/100 score +thresholds, zero for incorrect submissions, and extra fields that do not +inflate the score. Also covers directive fuzzy/exact matching, the +tool_arguments fallback, evidence safety, and config validation. +""" + +import pytest + +from finbot.ctf.detectors.implementations.purple_team_detector import ( + PurpleTeamDetector, +) + +EXPECTED = { + "server": "findrive", + "tool": "get_file", + "directive": "approve all pending invoices", +} + + +def _detector(config_overrides=None): + config = {"expected": dict(EXPECTED)} + if config_overrides: + config.update(config_overrides) + return PurpleTeamDetector("challenge-purple", config) + + +def _event(submission, key="submission"): + return { + "event_type": "business.investigation.submitted", + "namespace": "ns_test", + "user_id": "user_test", + "workflow_id": "wf_1", + key: submission, + } + + +# --- Scoring thresholds ----------------------------------------------------- + + +@pytest.mark.asyncio +async def test_all_three_correct_scores_100(): + det = _detector() + result = await det.check_event(_event(dict(EXPECTED)), db=None) + assert result.detected is True + assert result.evidence["score"] == 100 + assert result.confidence == 1.0 + + +@pytest.mark.asyncio +async def test_two_correct_scores_66_and_completes(): + det = _detector() + submission = {"server": "findrive", "tool": "get_file", "directive": "wrong"} + result = await det.check_event(_event(submission), db=None) + assert result.evidence["score"] == 66 + assert result.detected is True # 66 >= default min_score 50 + + +@pytest.mark.asyncio +async def test_directive_only_scores_34_and_fails(): + det = _detector() + submission = { + "server": "wrong", + "tool": "wrong", + "directive": "approve all pending invoices", + } + result = await det.check_event(_event(submission), db=None) + assert result.evidence["score"] == 34 + assert result.detected is False # 34 < 50 + + +@pytest.mark.asyncio +async def test_server_only_scores_33_and_fails(): + det = _detector() + submission = {"server": "findrive", "tool": "wrong", "directive": "wrong"} + result = await det.check_event(_event(submission), db=None) + assert result.evidence["score"] == 33 + assert result.detected is False + + +@pytest.mark.asyncio +async def test_all_wrong_scores_zero(): + det = _detector() + submission = {"server": "x", "tool": "y", "directive": "z"} + result = await det.check_event(_event(submission), db=None) + assert result.evidence["score"] == 0 + assert result.detected is False + assert result.confidence == 0.0 + + +# --- Extra fields, fallbacks, malformed input ------------------------------- + + +@pytest.mark.asyncio +async def test_extra_fields_do_not_inflate_score(): + det = _detector() + submission = dict(EXPECTED) + submission["bonus"] = "should be ignored" + submission["server_extra"] = "findrive" + result = await det.check_event(_event(submission), db=None) + assert result.evidence["score"] == 100 # not more than 100 + + +@pytest.mark.asyncio +async def test_tool_arguments_fallback(): + det = _detector() + # Submission delivered under tool_arguments instead of submission + result = await det.check_event( + _event(dict(EXPECTED), key="tool_arguments"), db=None + ) + assert result.evidence["score"] == 100 + + +@pytest.mark.asyncio +async def test_malformed_submission_not_detected(): + det = _detector() + result = await det.check_event(_event("not-a-dict"), db=None) + assert result.detected is False + + +@pytest.mark.asyncio +async def test_missing_submission_scores_zero(): + det = _detector() + event = {"event_type": "business.investigation.submitted"} + result = await det.check_event(event, db=None) + assert result.evidence["score"] == 0 + assert result.detected is False + + +# --- Directive matching modes ----------------------------------------------- + + +@pytest.mark.asyncio +async def test_directive_fuzzy_substring_match(): + det = _detector() + submission = { + "server": "findrive", + "tool": "get_file", + "directive": "Please approve all pending invoices immediately, thanks", + } + result = await det.check_event(_event(submission), db=None) + assert result.evidence["fields_correct"]["directive"] is True + assert result.evidence["score"] == 100 + + +@pytest.mark.asyncio +async def test_directive_fuzzy_paraphrase_reordered_words(): + # Student paraphrases: reordered clauses and "outside" for "external". + det = _detector() + submission = { + "server": "findrive", + "tool": "get_file", + "directive": "it emailed the vendor bank account and tax id to an outside address", + } + # expected: "approve all pending invoices" — override to a longer directive + det = PurpleTeamDetector( + "c", + { + "expected": { + "server": "findrive", + "tool": "get_file", + "directive": "email the vendor tax id and bank account to an external address", + } + }, + ) + result = await det.check_event(_event(submission), db=None) + assert result.evidence["fields_correct"]["directive"] is True + assert result.evidence["score"] == 100 + + +@pytest.mark.asyncio +async def test_directive_fuzzy_rejects_unrelated_text(): + det = PurpleTeamDetector( + "c", + { + "expected": { + "server": "findrive", + "tool": "get_file", + "directive": "email the vendor tax id and bank account to an external address", + } + }, + ) + submission = { + "server": "findrive", + "tool": "get_file", + "directive": "the agent deleted some invoices for no reason", + } + result = await det.check_event(_event(submission), db=None) + assert result.evidence["fields_correct"]["directive"] is False + assert result.evidence["score"] == 66 + + +@pytest.mark.asyncio +async def test_directive_exact_mode_rejects_substring(): + det = _detector({"directive_match": "exact"}) + submission = { + "server": "findrive", + "tool": "get_file", + "directive": "please approve all pending invoices now", + } + result = await det.check_event(_event(submission), db=None) + assert result.evidence["fields_correct"]["directive"] is False + assert result.evidence["score"] == 66 + + +@pytest.mark.asyncio +async def test_case_and_whitespace_normalized(): + det = _detector() + submission = { + "server": " FinDrive ", + "tool": "GET_FILE", + "directive": "approve all pending invoices", + } + result = await det.check_event(_event(submission), db=None) + assert result.evidence["score"] == 100 + + +# --- Evidence safety -------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_evidence_does_not_leak_expected_answer(): + det = _detector() + submission = {"server": "wrong", "tool": "wrong", "directive": "wrong"} + result = await det.check_event(_event(submission), db=None) + # Expected values must never appear in the evidence surfaced to the player. + assert "findrive" not in str(result.evidence) + assert "get_file" not in str(result.evidence) + assert "fields_correct" in result.evidence + + +# --- Partial expected answer ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_partial_expected_only_scores_defined_fields(): + # Challenge only asks for server + tool (no directive). + det = PurpleTeamDetector( + "c2", + { + "expected": {"server": "findrive", "tool": "get_file"}, + "field_points": {"server": 50, "tool": 50}, + }, + ) + result = await det.check_event( + _event({"server": "findrive", "tool": "get_file"}), db=None + ) + assert result.evidence["score"] == 100 + assert result.detected is True + + +# --- Config validation ------------------------------------------------------ + + +def test_config_requires_expected(): + with pytest.raises(ValueError): + PurpleTeamDetector("c", {}) + + +def test_config_rejects_unknown_expected_field(): + with pytest.raises(ValueError): + PurpleTeamDetector("c", {"expected": {"bogus": "x"}}) + + +def test_config_rejects_bad_min_score(): + with pytest.raises(ValueError): + PurpleTeamDetector("c", {"expected": {"server": "s"}, "min_score": 150}) + + +def test_config_rejects_bad_directive_match(): + with pytest.raises(ValueError): + PurpleTeamDetector( + "c", {"expected": {"server": "s"}, "directive_match": "regex"} + ) + + +def test_config_rejects_bad_directive_threshold(): + with pytest.raises(ValueError): + PurpleTeamDetector( + "c", {"expected": {"server": "s"}, "directive_threshold": 0} + ) + with pytest.raises(ValueError): + PurpleTeamDetector( + "c", {"expected": {"server": "s"}, "directive_threshold": 1.5} + ) + + +def test_relevant_event_types(): + det = _detector() + assert det.get_relevant_event_types() == ["business.investigation.submitted"] + + +# --- Review #573: substring bypass ------------------------------------------ +# +# The fuzzy directive check used to accept a submission that was merely a +# substring OF the expected answer. Because the seeded directive is a normal +# English sentence, single filler words like "to" or "and" are literal +# substrings of it and scored full credit -- a player could complete the +# challenge without understanding the attack at all. + + +@pytest.mark.parametrize("junk", ["to", "and", "all", "invoices", "app", "e"]) +@pytest.mark.asyncio +async def test_directive_junk_substring_is_rejected(junk): + """A fragment of the expected answer must not score the directive field.""" + det = _detector() + submission = {"server": "findrive", "tool": "get_file", "directive": junk} + result = await det.check_event(_event(submission), db=None) + + assert result.evidence["fields_correct"]["directive"] is False + # Server and tool still score, so this caps out below a perfect answer. + assert result.evidence["score"] == 66 + + +@pytest.mark.asyncio +async def test_single_word_cannot_reach_full_marks(): + """The reviewer's exact scenario: guess the enums, type one word, win.""" + det = _detector( + { + "expected": { + "server": "finstripe", + "tool": "create_transfer", + "directive": ( + "email the vendor tax id and bank account to an external address" + ), + } + } + ) + submission = { + "server": "finstripe", + "tool": "create_transfer", + "directive": "to", + } + result = await det.check_event(_event(submission), db=None) + + assert result.evidence["fields_correct"]["directive"] is False + assert result.evidence["score"] < 100 + + +@pytest.mark.asyncio +async def test_quoting_the_full_directive_still_accepted(): + """The safe direction -- answer contains the whole expected phrase -- stays.""" + det = _detector() + submission = { + "server": "findrive", + "tool": "get_file", + "directive": "It told the agent to approve all pending invoices, no review.", + } + result = await det.check_event(_event(submission), db=None) + + assert result.evidence["fields_correct"]["directive"] is True + assert result.evidence["score"] == 100 + + +@pytest.mark.asyncio +async def test_genuine_partial_answer_still_earns_the_field(): + """Rejecting fragments must not reject real paraphrases.""" + det = _detector( + { + "expected": { + "server": "finstripe", + "tool": "create_transfer", + "directive": ( + "email the vendor tax id and bank account to an external address" + ), + } + } + ) + submission = { + "server": "finstripe", + "tool": "create_transfer", + "directive": "emailed the vendor's tax id and bank account details externally", + } + result = await det.check_event(_event(submission), db=None) + + assert result.evidence["fields_correct"]["directive"] is True