{ "schema_version": "investigations-1.3", "dataset_version": "0.6.0", "status": "preliminary", "protocol_hash": "b91d80ea3be5920db093c35788aa6db549f7779bb81831896fbe22b0a69937a5", "cases": 48, "families": 16, "tiers": [ { "id": "basic", "cases": 36, "families": 12, "query_budget": 12, "record_budget": 40, "page_limit": 4 }, { "id": "intrusion", "cases": 12, "families": 4, "query_budget": 30, "record_budget": 150, "page_limit": 10 } ], "models": [ { "model_id": "anthropic/claude-sonnet-5.5", "label": "Anthropic: Claude Sonnet 5.5", "provider": "anthropic", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 91.66666666666666, "grounded_final_accuracy": 30.555555555555557, "grounded_checkpoint_accuracy": 60.18518518518519, "grounded_hypothesis_updates": 41.66666666666667, "workflow_completion": 100.0, "scope_precision": 97.22222222222221, "scope_recall": 100.0, "scope_exact": 94.44444444444444, "ioc_precision": 28.341836734693874, "ioc_recall": 90.625, "ioc_grounded_recall": 90.625, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 54.166666666666664, "false_separate_findings": 10, "sufficient_evidence_acquired": 47.22222222222222, "acquisition_milestone_recall": 74.07407407407408, "mean_query_calls": 11.277777777777779, "mean_returned_records": 17.944444444444443, "mean_latency_seconds": 47.047656860025505, "cost_usd": 12.51909, "cost_upper_bound_usd": 12.51909, "mean_cost_usd": 0.3477525, "mean_cost_upper_bound_usd": 0.3477525, "precision_cases": 28, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 83.33333333333334, "grounded_final_accuracy": 33.33333333333333, "grounded_checkpoint_accuracy": 52.77777777777778, "grounded_hypothesis_updates": 37.083333333333336, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 61.11111111111111, "scope_exact": 33.33333333333333, "ioc_precision": 6.25, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 6, "sufficient_evidence_acquired": 33.33333333333333, "acquisition_milestone_recall": 62.22222222222222, "mean_query_calls": 21.0, "mean_returned_records": 47.25, "mean_latency_seconds": 68.2594801179924, "cost_usd": 9.054066, "cost_upper_bound_usd": 9.054066, "mean_cost_usd": 0.7545055, "mean_cost_upper_bound_usd": 0.7545055, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 91.66666666666667, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 30.555555555555557, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 74.07407407407408, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 47.22222222222222, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 60.18518518518519, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 41.66666666666667, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 14.814814814814813, "applicable_cases": 36 }, { "id": "host_scope", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 28.341836734693878, "applicable_cases": 28 }, { "id": "ioc_grounded_recall", "score": 90.625, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 54.166666666666664, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 83.33333333333333, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": true, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 83.33333333333333, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 83.33333333333333, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": true, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 83.33333333333333, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 62.22222222222222, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 52.77777777777778, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 37.083333333333336, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 22.870370370370367, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 6.25, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 50.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 20.0, "host_scope": 0.0, "ioc_precision": 12.5, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 70.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 11.11111111111111, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": true, "evidence_backed": true }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] } ], "metrics": { "final_verdict_accuracy": "Correct final decision", "grounded_final_accuracy": "Final decision with proof", "scope_precision": "Correctly scoped hosts / reported hosts", "scope_recall": "Found scoped hosts / expected hosts", "ioc_precision": "Correct indicators / reported indicators", "ioc_recall": "Found indicators / expected indicators", "ioc_grounded_recall": "Indicators found with accepted proof / expected indicators", "benign_lookalikes_flagged": "Non-compromise artifacts incorrectly called indicators", "environment_grounded_recall": "Separate incidents found with proof / expected separate incidents", "false_separate_findings": "Independent incidents reported without a matching evidence key", "mean_latency_seconds": "Mean recorded time per case", "mean_cost_usd": "Mean recorded cost per case", "grounded_checkpoint_accuracy": "Correct evidence-backed decisions at sealed stages, averaged equally across cases", "grounded_hypothesis_updates": "Correct evidence-backed malicious and benign hypothesis updates", "workflow_completion": "Completed case workflows / all case attempts", "sufficient_evidence_acquired": "Cases where queries acquired the required proof / all case attempts" }, "release_hash": "ba4bc2aab466f11fa90a06f073f57401f7f5c67bdc1cbc575d84bad84d6a3042", "campaign_id": "dfir06-sonnet55-20261012", "campaign_hash": "fd929601a8d1423a8c0db4abac758e6dfe7665c66ee5047a11e87a6deb67722a", "generated_at": "2026-10-12T17:51:46.129021Z", "source_commits": [ "bb1485dcf6e65c7b8393454c59c00b7ea76a9cbb" ], "tactical_tasks": [ { "id": "final_decision", "label": "Reach the right decision", "description": "Correct final verdict, without requiring citations." }, { "id": "decision_with_proof", "label": "Support the final decision", "description": "Correct final verdict with an accepted set of evidence the agent retrieved." }, { "id": "evidence_milestones", "label": "Find evidence along the way", "description": "Fraction of required evidence milestones reached through tool queries." }, { "id": "sufficient_evidence", "label": "Gather sufficient evidence", "description": "Cases where queries acquired a complete accepted proof set, regardless of the final verdict." }, { "id": "stage_decisions", "label": "Assess evidence at each stage", "description": "Correct evidence-backed decisions at sealed checkpoints. Missing decisions receive zero." }, { "id": "hypothesis_updates", "label": "Test competing explanations", "description": "Correct evidence-backed updates to both malicious and benign hypotheses. Missing updates receive zero." }, { "id": "factual_reconstruction", "label": "Reconstruct the facts", "description": "Required factual findings correctly reported with accepted supporting evidence." }, { "id": "host_scope", "label": "Identify the exact host scope", "description": "Cases where the reported host set exactly matches the expected scope." }, { "id": "ioc_precision", "label": "Attribute indicators precisely", "description": "Correct indicators divided by reported indicators, averaged across cases reporting indicators. No reported indicators is not applicable." }, { "id": "ioc_grounded_recall", "label": "Find indicators with proof", "description": "Expected indicators found with accepted evidence, averaged across cases with expected indicators." }, { "id": "separate_incidents", "label": "Recognize separate incidents", "description": "Evidence-backed recall of other incidents, averaged across cases containing a separate incident." } ] }