{ "schema_version": "investigations-1.3", "dataset_version": "0.6.0", "status": "preliminary", "protocol_hash": "b91d80ea3be5920db093c35788aa6db549f7779bb81831896fbe22b0a69937a5", "cases": 48, "families": 16, "tiers": [ { "id": "basic", "cases": 36, "families": 12, "query_budget": 12, "record_budget": 40, "page_limit": 4 }, { "id": "intrusion", "cases": 12, "families": 4, "query_budget": 30, "record_budget": 150, "page_limit": 10 } ], "models": [ { "model_id": "meta/muse-wonder", "label": "Meta: Muse Wonder", "provider": "meta", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 75.0, "grounded_final_accuracy": 25.0, "grounded_checkpoint_accuracy": 62.03703703703704, "grounded_hypothesis_updates": 43.05555555555556, "workflow_completion": 97.22222222222221, "scope_precision": 100.0, "scope_recall": 97.22222222222221, "scope_exact": 97.22222222222221, "ioc_precision": 25.222222222222225, "ioc_recall": 75.0, "ioc_grounded_recall": 75.0, "benign_lookalikes_flagged": 1, "environment_grounded_recall": 8.333333333333332, "false_separate_findings": 10, "sufficient_evidence_acquired": 33.33333333333333, "acquisition_milestone_recall": 70.37037037037037, "mean_query_calls": 11.833333333333334, "mean_returned_records": 22.72222222222222, "mean_latency_seconds": 61.16998699686074, "cost_usd": 13.3139024, "cost_upper_bound_usd": 13.3139024, "mean_cost_usd": 0.36983062222222224, "mean_cost_upper_bound_usd": 0.36983062222222224, "precision_cases": 30, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 35, "invalid": 1 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 91.66666666666666, "grounded_final_accuracy": 33.33333333333333, "grounded_checkpoint_accuracy": 57.77777777777778, "grounded_hypothesis_updates": 44.02777777777778, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 77.77777777777779, "scope_exact": 66.66666666666666, "ioc_precision": 6.878306878306878, "ioc_recall": 100.0, "ioc_grounded_recall": 100.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 6, "sufficient_evidence_acquired": 58.333333333333336, "acquisition_milestone_recall": 86.38888888888889, "mean_query_calls": 26.75, "mean_returned_records": 89.16666666666667, "mean_latency_seconds": 96.37278711033893, "cost_usd": 9.3316232, "cost_upper_bound_usd": 9.3316232, "mean_cost_usd": 0.7776352666666666, "mean_cost_upper_bound_usd": 0.7776352666666666, "precision_cases": 9, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 75.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 25.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 70.37037037037037, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 33.333333333333336, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 62.03703703703704, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 43.05555555555556, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 18.518518518518515, "applicable_cases": 36 }, { "id": "host_scope", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 25.22222222222222, "applicable_cases": 30 }, { "id": "ioc_grounded_recall", "score": 75.0, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 8.333333333333334, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 91.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 86.38888888888889, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 58.333333333333336, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 57.77777777777778, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 44.02777777777778, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 34.97685185185185, "applicable_cases": 12 }, { "id": "host_scope", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 6.878306878306877, "applicable_cases": 9 }, { "id": "ioc_grounded_recall", "score": 100.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 20.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": true, "evidence_backed": true }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 40.0, "hypothesis_updates": 0.0, "factual_reconstruction": 37.5, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 40.0, "hypothesis_updates": 20.0, "factual_reconstruction": 55.55555555555556, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 30.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 50.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 40.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] } ], "metrics": { "final_verdict_accuracy": "Correct final decision", "grounded_final_accuracy": "Final decision with proof", "scope_precision": "Correctly scoped hosts / reported hosts", "scope_recall": "Found scoped hosts / expected hosts", "ioc_precision": "Correct indicators / reported indicators", "ioc_recall": "Found indicators / expected indicators", "ioc_grounded_recall": "Indicators found with accepted proof / expected indicators", "benign_lookalikes_flagged": "Non-compromise artifacts incorrectly called indicators", "environment_grounded_recall": "Separate incidents found with proof / expected separate incidents", "false_separate_findings": "Independent incidents reported without a matching evidence key", "mean_latency_seconds": "Mean recorded time per case", "mean_cost_usd": "Mean recorded cost per case", "grounded_checkpoint_accuracy": "Correct evidence-backed decisions at sealed stages, averaged equally across cases", "grounded_hypothesis_updates": "Correct evidence-backed malicious and benign hypothesis updates", "workflow_completion": "Completed case workflows / all case attempts", "sufficient_evidence_acquired": "Cases where queries acquired the required proof / all case attempts" }, "release_hash": "eb6ffd9f730782f058011f735615fd2b1ab9de5b88d2bcfba2939ae438bdcb32", "campaign_id": "dfir06-muse-wonder-20261016", "campaign_hash": "85aa250ca959267763ff32fef04b19573b2ce789ddabd9d51684e793d6fdff98", "generated_at": "2026-10-16T19:54:23.255478Z", "source_commits": [ "bd3a7084975d08f16e9e4853fce83f00c3c421a9" ], "tactical_tasks": [ { "id": "final_decision", "label": "Reach the right decision", "description": "Correct final verdict, without requiring citations." }, { "id": "decision_with_proof", "label": "Support the final decision", "description": "Correct final verdict with an accepted set of evidence the agent retrieved." }, { "id": "evidence_milestones", "label": "Find evidence along the way", "description": "Fraction of required evidence milestones reached through tool queries." }, { "id": "sufficient_evidence", "label": "Gather sufficient evidence", "description": "Cases where queries acquired a complete accepted proof set, regardless of the final verdict." }, { "id": "stage_decisions", "label": "Assess evidence at each stage", "description": "Correct evidence-backed decisions at sealed checkpoints. Missing decisions receive zero." }, { "id": "hypothesis_updates", "label": "Test competing explanations", "description": "Correct evidence-backed updates to both malicious and benign hypotheses. Missing updates receive zero." }, { "id": "factual_reconstruction", "label": "Reconstruct the facts", "description": "Required factual findings correctly reported with accepted supporting evidence." }, { "id": "host_scope", "label": "Identify the exact host scope", "description": "Cases where the reported host set exactly matches the expected scope." }, { "id": "ioc_precision", "label": "Attribute indicators precisely", "description": "Correct indicators divided by reported indicators, averaged across cases reporting indicators. No reported indicators is not applicable." }, { "id": "ioc_grounded_recall", "label": "Find indicators with proof", "description": "Expected indicators found with accepted evidence, averaged across cases with expected indicators." }, { "id": "separate_incidents", "label": "Recognize separate incidents", "description": "Evidence-backed recall of other incidents, averaged across cases containing a separate incident." } ] }