{ "schema_version": "investigations-1.3", "dataset_version": "0.6.0", "status": "preliminary", "protocol_hash": "b91d80ea3be5920db093c35788aa6db549f7779bb81831896fbe22b0a69937a5", "cases": 48, "families": 16, "tiers": [ { "id": "basic", "cases": 36, "families": 12, "query_budget": 12, "record_budget": 40, "page_limit": 4 }, { "id": "intrusion", "cases": 12, "families": 4, "query_budget": 30, "record_budget": 150, "page_limit": 10 } ], "models": [ { "model_id": "google/gemini-4-argon", "label": "Google: Gemini 4 Argon", "provider": "google-ai-studio", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 94.44444444444444, "grounded_final_accuracy": 80.55555555555556, "grounded_checkpoint_accuracy": 87.03703703703704, "grounded_hypothesis_updates": 65.27777777777779, "workflow_completion": 97.22222222222221, "scope_precision": 98.57142857142858, "scope_recall": 97.22222222222221, "scope_exact": 94.44444444444444, "ioc_precision": 32.21904761904762, "ioc_recall": 100.0, "ioc_grounded_recall": 100.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 79.16666666666666, "false_separate_findings": 5, "sufficient_evidence_acquired": 97.22222222222221, "acquisition_milestone_recall": 99.07407407407408, "mean_query_calls": 11.472222222222221, "mean_returned_records": 22.166666666666668, "mean_latency_seconds": 380.169925181029, "cost_usd": 13.5168837, "cost_upper_bound_usd": 15.4100277, "mean_cost_usd": 0.37546899166666664, "mean_cost_upper_bound_usd": 0.428056325, "precision_cases": 25, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 35, "invalid": 1 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 83.33333333333334, "grounded_final_accuracy": 33.33333333333333, "grounded_checkpoint_accuracy": 65.83333333333333, "grounded_hypothesis_updates": 49.861111111111114, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 77.77777777777779, "scope_exact": 66.66666666666666, "ioc_precision": 4.166666666666666, "ioc_recall": 75.0, "ioc_grounded_recall": 75.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 100.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 66.66666666666666, "acquisition_milestone_recall": 91.38888888888889, "mean_query_calls": 25.583333333333332, "mean_returned_records": 103.16666666666667, "mean_latency_seconds": 584.4349046844048, "cost_usd": 8.1763943, "cost_upper_bound_usd": 8.1763943, "mean_cost_usd": 0.6813661916666667, "mean_cost_upper_bound_usd": 0.6813661916666667, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 80.55555555555556, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 99.07407407407408, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 87.03703703703704, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 65.27777777777779, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 20.370370370370367, "applicable_cases": 36 }, { "id": "host_scope", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 32.21904761904762, "applicable_cases": 25 }, { "id": "ioc_grounded_recall", "score": 100.0, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 79.16666666666667, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 83.33333333333333, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 83.33333333333333, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 91.38888888888889, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 65.83333333333333, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 49.861111111111114, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 29.305555555555554, "applicable_cases": 12 }, { "id": "host_scope", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 4.166666666666666, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 75.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 100.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 20.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": true, "evidence_backed": true }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 41.666666666666664, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 60.0, "hypothesis_updates": 20.0, "factual_reconstruction": 25.0, "host_scope": 100.0, "ioc_precision": 16.666666666666664, "ioc_grounded_recall": 50.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 80.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 60.0, "hypothesis_updates": 40.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 40.0, "hypothesis_updates": 30.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 40.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] } ], "metrics": { "final_verdict_accuracy": "Correct final decision", "grounded_final_accuracy": "Final decision with proof", "scope_precision": "Correctly scoped hosts / reported hosts", "scope_recall": "Found scoped hosts / expected hosts", "ioc_precision": "Correct indicators / reported indicators", "ioc_recall": "Found indicators / expected indicators", "ioc_grounded_recall": "Indicators found with accepted proof / expected indicators", "benign_lookalikes_flagged": "Non-compromise artifacts incorrectly called indicators", "environment_grounded_recall": "Separate incidents found with proof / expected separate incidents", "false_separate_findings": "Independent incidents reported without a matching evidence key", "mean_latency_seconds": "Mean recorded time per case", "mean_cost_usd": "Mean recorded cost per case", "grounded_checkpoint_accuracy": "Correct evidence-backed decisions at sealed stages, averaged equally across cases", "grounded_hypothesis_updates": "Correct evidence-backed malicious and benign hypothesis updates", "workflow_completion": "Completed case workflows / all case attempts", "sufficient_evidence_acquired": "Cases where queries acquired the required proof / all case attempts" }, "release_hash": "baec2bbe8f3e38a53b60539c8824f11878a296ae49af5a4c6f109e3029f420fd", "campaign_id": "dfir06-gemini4argon-studio-20261014", "campaign_hash": "0746748aff62bfc00766fab6afbb8d61bce4d753d7f9a256310c2c014468f85d", "generated_at": "2026-10-15T10:56:58.860980Z", "source_commits": [ "97c3c47f592cd7683a79e3c32daa19e363be97d6" ], "tactical_tasks": [ { "id": "final_decision", "label": "Reach the right decision", "description": "Correct final verdict, without requiring citations." }, { "id": "decision_with_proof", "label": "Support the final decision", "description": "Correct final verdict with an accepted set of evidence the agent retrieved." }, { "id": "evidence_milestones", "label": "Find evidence along the way", "description": "Fraction of required evidence milestones reached through tool queries." }, { "id": "sufficient_evidence", "label": "Gather sufficient evidence", "description": "Cases where queries acquired a complete accepted proof set, regardless of the final verdict." }, { "id": "stage_decisions", "label": "Assess evidence at each stage", "description": "Correct evidence-backed decisions at sealed checkpoints. Missing decisions receive zero." }, { "id": "hypothesis_updates", "label": "Test competing explanations", "description": "Correct evidence-backed updates to both malicious and benign hypotheses. Missing updates receive zero." }, { "id": "factual_reconstruction", "label": "Reconstruct the facts", "description": "Required factual findings correctly reported with accepted supporting evidence." }, { "id": "host_scope", "label": "Identify the exact host scope", "description": "Cases where the reported host set exactly matches the expected scope." }, { "id": "ioc_precision", "label": "Attribute indicators precisely", "description": "Correct indicators divided by reported indicators, averaged across cases reporting indicators. No reported indicators is not applicable." }, { "id": "ioc_grounded_recall", "label": "Find indicators with proof", "description": "Expected indicators found with accepted evidence, averaged across cases with expected indicators." }, { "id": "separate_incidents", "label": "Recognize separate incidents", "description": "Evidence-backed recall of other incidents, averaged across cases containing a separate incident." } ] }