{ "schema_version": "investigations-1.3", "dataset_version": "0.6.0", "status": "preliminary", "protocol_hash": "b91d80ea3be5920db093c35788aa6db549f7779bb81831896fbe22b0a69937a5", "cases": 48, "families": 16, "tiers": [ { "id": "basic", "cases": 36, "families": 12, "query_budget": 12, "record_budget": 40, "page_limit": 4 }, { "id": "intrusion", "cases": 12, "families": 4, "query_budget": 30, "record_budget": 150, "page_limit": 10 } ], "models": [ { "model_id": "deepseek/deepseek-v4-flash", "label": "DeepSeek: DeepSeek V4 Flash 0423", "provider": "deepinfra/fp8", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 36.11111111111111, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 28.703703703703706, "grounded_hypothesis_updates": 18.981481481481485, "workflow_completion": 94.44444444444444, "scope_precision": 96.32352941176471, "scope_recall": 94.44444444444444, "scope_exact": 88.88888888888889, "ioc_precision": 8.974358974358974, "ioc_recall": 21.875, "ioc_grounded_recall": 21.875, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 22.916666666666664, "false_separate_findings": 6, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 33.33333333333333, "mean_query_calls": 12.0, "mean_returned_records": 10.61111111111111, "mean_latency_seconds": 354.98373651265706, "cost_usd": 0.404155854, "cost_upper_bound_usd": 1.147378158, "mean_cost_usd": 0.0112265515, "mean_cost_upper_bound_usd": 0.0318716155, "precision_cases": 26, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 34, "turn_limit": 2 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 58.333333333333336, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 13.88888888888889, "grounded_hypothesis_updates": 11.527777777777779, "workflow_completion": 100.0, "scope_precision": 94.44444444444444, "scope_recall": 55.55555555555556, "scope_exact": 33.33333333333333, "ioc_precision": 6.802721088435374, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 42.857142857142854, "false_separate_findings": 2, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 21.11111111111111, "mean_query_calls": 30.0, "mean_returned_records": 40.25, "mean_latency_seconds": 332.86214802758576, "cost_usd": 0.238724406, "cost_upper_bound_usd": 0.35181369, "mean_cost_usd": 0.0198937005, "mean_cost_upper_bound_usd": 0.0293178075, "precision_cases": 7, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 36.111111111111114, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 33.33333333333333, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 28.703703703703706, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 18.981481481481485, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 4.629629629629629, "applicable_cases": 36 }, { "id": "host_scope", "score": 88.88888888888889, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 8.974358974358974, "applicable_cases": 26 }, { "id": "ioc_grounded_recall", "score": 21.875, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 22.916666666666668, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 50.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 58.333333333333336, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 21.11111111111111, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 13.88888888888889, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 11.527777777777779, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 6.388888888888888, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 6.802721088435374, "applicable_cases": 7 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 42.857142857142854, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 14.285714285714285, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "z-ai/glm-5.3-flash", "label": "Z.ai: GLM 5.3 Flash", "provider": "z-ai/fp8", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 50.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 29.629629629629633, "grounded_hypothesis_updates": 24.074074074074076, "workflow_completion": 100.0, "scope_precision": 98.61111111111111, "scope_recall": 100.0, "scope_exact": 97.22222222222221, "ioc_precision": 7.8125, "ioc_recall": 31.25, "ioc_grounded_recall": 31.25, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 25.0, "false_separate_findings": 11, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 37.96296296296296, "mean_query_calls": 12.0, "mean_returned_records": 20.416666666666668, "mean_latency_seconds": 255.7349191078085, "cost_usd": 0.60861596, "cost_upper_bound_usd": 0.60861596, "mean_cost_usd": 0.016905998888888888, "mean_cost_upper_bound_usd": 0.016905998888888888, "precision_cases": 32, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 91.66666666666666, "grounded_final_accuracy": 25.0, "grounded_checkpoint_accuracy": 45.833333333333336, "grounded_hypothesis_updates": 34.583333333333336, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 63.888888888888886, "scope_exact": 41.66666666666667, "ioc_precision": 1.7857142857142856, "ioc_recall": 25.0, "ioc_grounded_recall": 25.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 7, "sufficient_evidence_acquired": 33.33333333333333, "acquisition_milestone_recall": 68.61111111111111, "mean_query_calls": 27.416666666666668, "mean_returned_records": 102.83333333333333, "mean_latency_seconds": 372.6755758718161, "cost_usd": 0.46299744, "cost_upper_bound_usd": 0.46299744, "mean_cost_usd": 0.03858312, "mean_cost_upper_bound_usd": 0.03858312, "precision_cases": 8, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 50.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 37.962962962962955, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 29.629629629629633, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 24.074074074074076, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 6.481481481481481, "applicable_cases": 36 }, { "id": "host_scope", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 7.8125, "applicable_cases": 32 }, { "id": "ioc_grounded_recall", "score": 31.25, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 25.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 66.66666666666666, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 91.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 25.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 68.61111111111111, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 45.833333333333336, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 34.583333333333336, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 2.7777777777777772, "applicable_cases": 12 }, { "id": "host_scope", "score": 41.666666666666664, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 1.7857142857142856, "applicable_cases": 8 }, { "id": "ioc_grounded_recall", "score": 25.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 14.285714285714285, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 60.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 60.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 60.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "minimax/minimax-m3", "label": "MiniMax: MiniMax M3", "provider": "deepinfra/fp8", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 0.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 0.0, "grounded_hypothesis_updates": 0.0, "workflow_completion": 0.0, "scope_precision": null, "scope_recall": 0.0, "scope_exact": 0.0, "ioc_precision": null, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 33.33333333333333, "mean_query_calls": 11.527777777777779, "mean_returned_records": 14.86111111111111, "mean_latency_seconds": 477.27941844217844, "cost_usd": 1.251021416, "cost_upper_bound_usd": 2.4915037040000003, "mean_cost_usd": 0.03475059488888889, "mean_cost_upper_bound_usd": 0.06920843622222222, "precision_cases": 0, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "invalid": 34, "context_limit": 1, "turn_limit": 1 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 0.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 0.0, "grounded_hypothesis_updates": 0.0, "workflow_completion": 0.0, "scope_precision": null, "scope_recall": 0.0, "scope_exact": 0.0, "ioc_precision": null, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 18.333333333333336, "mean_query_calls": 22.166666666666668, "mean_returned_records": 44.416666666666664, "mean_latency_seconds": 356.83825290292344, "cost_usd": 0.595258688, "cost_upper_bound_usd": 0.632892608, "mean_cost_usd": 0.04960489066666667, "mean_cost_upper_bound_usd": 0.05274105066666667, "precision_cases": 0, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "invalid": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 0.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 33.33333333333333, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 0.0, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 0.0, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 36 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 36 }, { "id": "ioc_precision", "score": null, "applicable_cases": 0 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 0.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 18.333333333333332, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 0.0, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 0.0, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 12 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": null, "applicable_cases": 0 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-6-luna", "label": "OpenAI: GPT-6 Luna", "provider": "openai/flex", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 63.888888888888886, "grounded_final_accuracy": 5.555555555555555, "grounded_checkpoint_accuracy": 45.370370370370374, "grounded_hypothesis_updates": 30.092592592592595, "workflow_completion": 97.22222222222221, "scope_precision": 100.0, "scope_recall": 97.22222222222221, "scope_exact": 97.22222222222221, "ioc_precision": 24.935897435897438, "ioc_recall": 53.125, "ioc_grounded_recall": 53.125, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 12.5, "false_separate_findings": 6, "sufficient_evidence_acquired": 13.88888888888889, "acquisition_milestone_recall": 61.11111111111111, "mean_query_calls": 11.833333333333334, "mean_returned_records": 18.52777777777778, "mean_latency_seconds": 72.05493408011938, "cost_usd": 0.1569376775, "cost_upper_bound_usd": 0.1569376775, "mean_cost_usd": 0.004359379930555556, "mean_cost_upper_bound_usd": 0.004359379930555556, "precision_cases": 26, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 35, "invalid": 1 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 75.0, "grounded_final_accuracy": 8.333333333333332, "grounded_checkpoint_accuracy": 47.5, "grounded_hypothesis_updates": 38.88888888888889, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 61.11111111111111, "scope_exact": 41.66666666666667, "ioc_precision": 3.3333333333333335, "ioc_recall": 16.666666666666664, "ioc_grounded_recall": 16.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 5, "sufficient_evidence_acquired": 25.0, "acquisition_milestone_recall": 57.49999999999999, "mean_query_calls": 23.916666666666668, "mean_returned_records": 54.416666666666664, "mean_latency_seconds": 128.59510319515053, "cost_usd": 0.0958023675, "cost_upper_bound_usd": 0.0958023675, "mean_cost_usd": 0.007983530625, "mean_cost_upper_bound_usd": 0.007983530625, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 63.888888888888886, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 5.555555555555555, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 61.11111111111111, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 13.88888888888889, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 45.370370370370374, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 30.092592592592595, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 14.814814814814813, "applicable_cases": 36 }, { "id": "host_scope", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 24.935897435897434, "applicable_cases": 26 }, { "id": "ioc_grounded_recall", "score": 53.125, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 12.5, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 75.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 8.333333333333334, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 57.5, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 25.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 47.5, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 38.88888888888889, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 18.333333333333332, "applicable_cases": 12 }, { "id": "host_scope", "score": 41.666666666666664, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 3.333333333333333, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 16.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 20.0, "host_scope": 0.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 25.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 20.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "google/gemini-3.8-flash", "label": "Google: Gemini 3.8 Flash", "provider": "google-ai-studio/flex", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 50.0, "grounded_final_accuracy": 2.7777777777777777, "grounded_checkpoint_accuracy": 33.333333333333336, "grounded_hypothesis_updates": 32.40740740740741, "workflow_completion": 86.11111111111111, "scope_precision": 100.0, "scope_recall": 86.11111111111111, "scope_exact": 86.11111111111111, "ioc_precision": 19.940476190476193, "ioc_recall": 43.75, "ioc_grounded_recall": 43.75, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 50.0, "false_separate_findings": 7, "sufficient_evidence_acquired": 8.333333333333332, "acquisition_milestone_recall": 45.37037037037037, "mean_query_calls": 11.722222222222221, "mean_returned_records": 22.305555555555557, "mean_latency_seconds": 102.56932759795583, "cost_usd": 4.2209997, "cost_upper_bound_usd": 4.2209997, "mean_cost_usd": 0.11724999166666666, "mean_cost_upper_bound_usd": 0.11724999166666666, "precision_cases": 24, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 31, "invalid": 5 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 91.66666666666666, "grounded_final_accuracy": 33.33333333333333, "grounded_checkpoint_accuracy": 46.666666666666664, "grounded_hypothesis_updates": 41.388888888888886, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 58.333333333333336, "scope_exact": 33.33333333333333, "ioc_precision": 7.777777777777778, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 6, "sufficient_evidence_acquired": 33.33333333333333, "acquisition_milestone_recall": 65.27777777777779, "mean_query_calls": 28.333333333333332, "mean_returned_records": 116.83333333333333, "mean_latency_seconds": 156.17786670537316, "cost_usd": 2.533715925, "cost_upper_bound_usd": 2.533715925, "mean_cost_usd": 0.21114299375, "mean_cost_upper_bound_usd": 0.21114299375, "precision_cases": 9, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 50.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 45.37037037037037, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 8.333333333333334, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 33.333333333333336, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 32.40740740740741, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 10.185185185185183, "applicable_cases": 36 }, { "id": "host_scope", "score": 86.11111111111111, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 19.94047619047619, "applicable_cases": 24 }, { "id": "ioc_grounded_recall", "score": 43.75, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 50.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 91.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 65.27777777777777, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 46.666666666666664, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 41.388888888888886, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 11.041666666666666, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 7.777777777777778, "applicable_cases": 9 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 20.0, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 12.5, "host_scope": 0.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 40.0, "hypothesis_updates": 30.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 80.0, "hypothesis_updates": 80.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 20.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "qwen/qwen3.7-flash", "label": "Qwen: Qwen3.7 Flash", "provider": "alibaba", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 52.77777777777778, "grounded_final_accuracy": 2.7777777777777777, "grounded_checkpoint_accuracy": 25.925925925925927, "grounded_hypothesis_updates": 15.740740740740742, "workflow_completion": 83.33333333333334, "scope_precision": 98.33333333333333, "scope_recall": 83.33333333333334, "scope_exact": 80.55555555555556, "ioc_precision": 22.348484848484848, "ioc_recall": 40.625, "ioc_grounded_recall": 40.625, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 33.33333333333333, "false_separate_findings": 5, "sufficient_evidence_acquired": 5.555555555555555, "acquisition_milestone_recall": 38.88888888888889, "mean_query_calls": 10.722222222222221, "mean_returned_records": 21.02777777777778, "mean_latency_seconds": 174.27120155588716, "cost_usd": 0.250918518, "cost_upper_bound_usd": 0.5376391979999999, "mean_cost_usd": 0.006969958833333333, "mean_cost_upper_bound_usd": 0.014934422166666664, "precision_cases": 22, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 30, "turn_limit": 6 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 66.66666666666666, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 8.88888888888889, "grounded_hypothesis_updates": 6.527777777777779, "workflow_completion": 100.0, "scope_precision": 97.22222222222221, "scope_recall": 58.333333333333336, "scope_exact": 33.33333333333333, "ioc_precision": 13.88888888888889, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 4, "sufficient_evidence_acquired": 16.666666666666664, "acquisition_milestone_recall": 40.0, "mean_query_calls": 16.416666666666668, "mean_returned_records": 46.75, "mean_latency_seconds": 227.5792774426227, "cost_usd": 0.200181452, "cost_upper_bound_usd": 0.5171670319999999, "mean_cost_usd": 0.016681787666666666, "mean_cost_upper_bound_usd": 0.04309725266666666, "precision_cases": 6, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 52.77777777777778, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 38.888888888888886, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 5.555555555555555, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 25.925925925925927, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 15.740740740740742, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 3.7037037037037033, "applicable_cases": 36 }, { "id": "host_scope", "score": 80.55555555555556, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 22.348484848484848, "applicable_cases": 22 }, { "id": "ioc_grounded_recall", "score": 40.625, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 33.333333333333336, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 40.0, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 8.88888888888889, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 6.527777777777779, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 0.8333333333333334, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 13.888888888888888, "applicable_cases": 6 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-oss-120b", "label": "OpenAI: gpt-oss-120b", "provider": "deepinfra/bf16", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 0.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 4.62962962962963, "grounded_hypothesis_updates": 4.62962962962963, "workflow_completion": 0.0, "scope_precision": null, "scope_recall": 0.0, "scope_exact": 0.0, "ioc_precision": null, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 33.33333333333333, "mean_query_calls": 3.361111111111111, "mean_returned_records": 2.0555555555555554, "mean_latency_seconds": 196.91585183347019, "cost_usd": 0.091287483, "cost_upper_bound_usd": 0.09697063139999999, "mean_cost_usd": 0.002535763416666667, "mean_cost_upper_bound_usd": 0.00269362865, "precision_cases": 0, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "invalid": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 0.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 1.6666666666666667, "grounded_hypothesis_updates": 1.6666666666666667, "workflow_completion": 0.0, "scope_precision": null, "scope_recall": 0.0, "scope_exact": 0.0, "ioc_precision": null, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 18.333333333333336, "mean_query_calls": 2.6666666666666665, "mean_returned_records": 1.4166666666666667, "mean_latency_seconds": 47.556414257356664, "cost_usd": 0.022390311, "cost_upper_bound_usd": 0.022390311, "mean_cost_usd": 0.00186585925, "mean_cost_upper_bound_usd": 0.00186585925, "precision_cases": 0, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "invalid": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 0.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 33.33333333333333, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 4.62962962962963, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 4.62962962962963, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 2.7777777777777772, "applicable_cases": 36 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 36 }, { "id": "ioc_precision", "score": null, "applicable_cases": 0 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 0.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 18.333333333333332, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 1.6666666666666667, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 1.6666666666666667, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 2.7777777777777772, "applicable_cases": 12 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": null, "applicable_cases": 0 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "poolside/laguna-s-2.1", "label": "Poolside: Laguna S 2.1", "provider": "poolside/fp4", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 8.333333333333332, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 11.111111111111112, "grounded_hypothesis_updates": 5.092592592592593, "workflow_completion": 13.88888888888889, "scope_precision": 90.0, "scope_recall": 13.88888888888889, "scope_exact": 11.11111111111111, "ioc_precision": 25.0, "ioc_recall": 6.25, "ioc_grounded_recall": 6.25, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 2, "sufficient_evidence_acquired": 2.7777777777777777, "acquisition_milestone_recall": 35.18518518518518, "mean_query_calls": 11.5, "mean_returned_records": 9.055555555555555, "mean_latency_seconds": 503.41671255547163, "cost_usd": 0.565880004, "cost_upper_bound_usd": 0.565880004, "mean_cost_usd": 0.015718889, "mean_cost_upper_bound_usd": 0.015718889, "precision_cases": 4, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 5, "invalid": 10, "context_limit": 20, "turn_limit": 1 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 0.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 5.833333333333334, "grounded_hypothesis_updates": 5.833333333333334, "workflow_completion": 0.0, "scope_precision": null, "scope_recall": 0.0, "scope_exact": 0.0, "ioc_precision": null, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 21.11111111111111, "mean_query_calls": 23.416666666666668, "mean_returned_records": 28.333333333333332, "mean_latency_seconds": 609.3977652502459, "cost_usd": 0.215371116, "cost_upper_bound_usd": 0.241373844, "mean_cost_usd": 0.017947593, "mean_cost_upper_bound_usd": 0.020114487, "precision_cases": 0, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "context_limit": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 8.333333333333334, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 35.18518518518518, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 11.111111111111112, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 5.092592592592593, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 4.629629629629629, "applicable_cases": 36 }, { "id": "host_scope", "score": 11.11111111111111, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 25.0, "applicable_cases": 4 }, { "id": "ioc_grounded_recall", "score": 6.25, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 0.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 21.11111111111111, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 5.833333333333334, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 5.833333333333334, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 0.8333333333333334, "applicable_cases": 12 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": null, "applicable_cases": 0 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-5-nano", "label": "OpenAI: GPT-5 Nano", "provider": "openai/flex", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 2.7777777777777777, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 21.296296296296298, "grounded_hypothesis_updates": 16.666666666666668, "workflow_completion": 19.444444444444446, "scope_precision": 55.952380952380956, "scope_recall": 19.444444444444446, "scope_exact": 5.555555555555555, "ioc_precision": 15.0, "ioc_recall": 6.25, "ioc_grounded_recall": 6.25, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 5, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 33.33333333333333, "mean_query_calls": 5.722222222222222, "mean_returned_records": 4.694444444444445, "mean_latency_seconds": 127.79078981983993, "cost_usd": 0.231762515, "cost_upper_bound_usd": 0.231762515, "mean_cost_usd": 0.006437847638888889, "mean_cost_upper_bound_usd": 0.006437847638888889, "precision_cases": 5, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "invalid": 29, "completed": 7 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 8.333333333333332, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 16.666666666666668, "grounded_hypothesis_updates": 13.61111111111111, "workflow_completion": 41.66666666666667, "scope_precision": 60.0, "scope_recall": 13.88888888888889, "scope_exact": 0.0, "ioc_precision": 0.0, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 1, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 20.0, "mean_query_calls": 10.083333333333334, "mean_returned_records": 10.666666666666666, "mean_latency_seconds": 138.69052489279906, "cost_usd": 0.091678785, "cost_upper_bound_usd": 0.091678785, "mean_cost_usd": 0.00763989875, "mean_cost_upper_bound_usd": 0.00763989875, "precision_cases": 1, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "invalid": 7, "completed": 5 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 33.33333333333333, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 21.296296296296298, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 16.666666666666668, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 36 }, { "id": "host_scope", "score": 5.555555555555555, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 15.0, "applicable_cases": 5 }, { "id": "ioc_grounded_recall", "score": 6.25, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 8.333333333333334, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 20.0, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 13.61111111111111, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 12 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 0.0, "applicable_cases": 1 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-4o-mini", "label": "OpenAI: GPT-4o-mini", "provider": "openai", "effort": null, "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 11.11111111111111, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 5.555555555555556, "grounded_hypothesis_updates": 5.092592592592593, "workflow_completion": 16.666666666666664, "scope_precision": 100.0, "scope_recall": 16.666666666666664, "scope_exact": 16.666666666666664, "ioc_precision": 0.0, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 33.33333333333333, "mean_query_calls": 4.472222222222222, "mean_returned_records": 4.861111111111111, "mean_latency_seconds": 55.63798345763644, "cost_usd": 0.8145152999999999, "cost_upper_bound_usd": 0.8145152999999999, "mean_cost_usd": 0.022625424999999998, "mean_cost_upper_bound_usd": 0.022625424999999998, "precision_cases": 1, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "turn_limit": 22, "invalid": 8, "completed": 6 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 0.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 1.6666666666666667, "grounded_hypothesis_updates": 1.6666666666666667, "workflow_completion": 8.333333333333332, "scope_precision": 100.0, "scope_recall": 2.7777777777777777, "scope_exact": 0.0, "ioc_precision": null, "ioc_recall": 0.0, "ioc_grounded_recall": 0.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 18.333333333333336, "mean_query_calls": 6.5, "mean_returned_records": 10.166666666666666, "mean_latency_seconds": 90.35226315185234, "cost_usd": 0.55103085, "cost_upper_bound_usd": 0.55103085, "mean_cost_usd": 0.045919237499999994, "mean_cost_upper_bound_usd": 0.045919237499999994, "precision_cases": 0, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "invalid": 4, "context_limit": 7, "completed": 1 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 11.11111111111111, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 33.33333333333333, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 5.555555555555556, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 5.092592592592593, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 36 }, { "id": "host_scope", "score": 16.666666666666668, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 0.0, "applicable_cases": 1 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 0.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 18.333333333333332, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 1.6666666666666667, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 1.6666666666666667, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 12 }, { "id": "host_scope", "score": 0.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": null, "applicable_cases": 0 }, { "id": "ioc_grounded_recall", "score": 0.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "anthropic/claude-haiku-4.5", "label": "Anthropic: Claude Haiku 4.5", "provider": "anthropic", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 50.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 21.296296296296298, "grounded_hypothesis_updates": 15.740740740740742, "workflow_completion": 94.44444444444444, "scope_precision": 98.52941176470588, "scope_recall": 94.44444444444444, "scope_exact": 91.66666666666666, "ioc_precision": 12.179487179487179, "ioc_recall": 37.5, "ioc_grounded_recall": 37.5, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 29.166666666666668, "false_separate_findings": 1, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 34.25925925925925, "mean_query_calls": 11.972222222222221, "mean_returned_records": 17.97222222222222, "mean_latency_seconds": 110.31162677100252, "cost_usd": 11.02616, "cost_upper_bound_usd": 11.02616, "mean_cost_usd": 0.30628222222222223, "mean_cost_upper_bound_usd": 0.30628222222222223, "precision_cases": 26, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 34, "invalid": 1, "turn_limit": 1 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 50.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 13.333333333333334, "grounded_hypothesis_updates": 11.25, "workflow_completion": 100.0, "scope_precision": 77.08333333333334, "scope_recall": 55.55555555555556, "scope_exact": 25.0, "ioc_precision": 4.583333333333333, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 5, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 42.22222222222222, "mean_query_calls": 29.833333333333332, "mean_returned_records": 84.66666666666667, "mean_latency_seconds": 198.99088565940232, "cost_usd": 10.297212, "cost_upper_bound_usd": 10.297212, "mean_cost_usd": 0.858101, "mean_cost_upper_bound_usd": 0.858101, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 50.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 34.25925925925925, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 21.296296296296298, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 15.740740740740742, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 6.481481481481481, "applicable_cases": 36 }, { "id": "host_scope", "score": 91.66666666666667, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 12.179487179487179, "applicable_cases": 26 }, { "id": "ioc_grounded_recall", "score": 37.5, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 29.166666666666668, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "turn_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 50.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 42.22222222222222, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 13.333333333333334, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 11.25, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 5.37037037037037, "applicable_cases": 12 }, { "id": "host_scope", "score": 25.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 4.583333333333333, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 12.5, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 11.11111111111111, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 50.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-5.6-luna", "label": "OpenAI: GPT-5.6 Luna", "provider": "openai/flex", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 83.33333333333334, "grounded_final_accuracy": 5.555555555555555, "grounded_checkpoint_accuracy": 44.44444444444445, "grounded_hypothesis_updates": 29.166666666666668, "workflow_completion": 100.0, "scope_precision": 97.22222222222221, "scope_recall": 100.0, "scope_exact": 94.44444444444444, "ioc_precision": 30.704365079365083, "ioc_recall": 68.75, "ioc_grounded_recall": 68.75, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 4.166666666666666, "false_separate_findings": 8, "sufficient_evidence_acquired": 13.88888888888889, "acquisition_milestone_recall": 62.96296296296296, "mean_query_calls": 11.555555555555555, "mean_returned_records": 15.194444444444445, "mean_latency_seconds": 88.84970039131683, "cost_usd": 0.26018392, "cost_upper_bound_usd": 0.26018392, "mean_cost_usd": 0.007227331111111112, "mean_cost_upper_bound_usd": 0.007227331111111112, "precision_cases": 24, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 100.0, "grounded_final_accuracy": 8.333333333333332, "grounded_checkpoint_accuracy": 32.22222222222222, "grounded_hypothesis_updates": 28.47222222222222, "workflow_completion": 100.0, "scope_precision": 95.83333333333334, "scope_recall": 58.333333333333336, "scope_exact": 33.33333333333333, "ioc_precision": 3.125, "ioc_recall": 16.666666666666664, "ioc_grounded_recall": 16.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 5, "sufficient_evidence_acquired": 25.0, "acquisition_milestone_recall": 59.44444444444444, "mean_query_calls": 24.25, "mean_returned_records": 52.666666666666664, "mean_latency_seconds": 108.33182913857551, "cost_usd": 0.15132945, "cost_upper_bound_usd": 0.15132945, "mean_cost_usd": 0.0126107875, "mean_cost_upper_bound_usd": 0.0126107875, "precision_cases": 8, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 83.33333333333333, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 5.555555555555555, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 62.962962962962955, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 13.88888888888889, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 44.44444444444445, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 29.166666666666668, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 16.666666666666664, "applicable_cases": 36 }, { "id": "host_scope", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 30.70436507936508, "applicable_cases": 24 }, { "id": "ioc_grounded_recall", "score": 68.75, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 4.166666666666667, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 16.666666666666664, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 100.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 8.333333333333334, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 59.44444444444444, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 25.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 32.22222222222222, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 28.47222222222222, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 17.407407407407405, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 3.125, "applicable_cases": 8 }, { "id": "ioc_grounded_recall", "score": 16.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 25.0, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 58.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 80.0, "hypothesis_updates": 70.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 22.22222222222222, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 80.0, "hypothesis_updates": 80.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 50.0, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "anthropic/claude-opus-5.5", "label": "Anthropic: Claude Opus 5.5", "provider": "anthropic", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 91.66666666666666, "grounded_final_accuracy": 33.33333333333333, "grounded_checkpoint_accuracy": 58.333333333333336, "grounded_hypothesis_updates": 34.72222222222222, "workflow_completion": 100.0, "scope_precision": 97.22222222222221, "scope_recall": 100.0, "scope_exact": 94.44444444444444, "ioc_precision": 30.62610229276896, "ioc_recall": 96.875, "ioc_grounded_recall": 96.875, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 29.166666666666668, "false_separate_findings": 8, "sufficient_evidence_acquired": 36.11111111111111, "acquisition_milestone_recall": 72.22222222222221, "mean_query_calls": 11.833333333333334, "mean_returned_records": 15.777777777777779, "mean_latency_seconds": 74.94654153696804, "cost_usd": 30.28936, "cost_upper_bound_usd": 30.28936, "mean_cost_usd": 0.8413711111111111, "mean_cost_upper_bound_usd": 0.8413711111111111, "precision_cases": 27, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 100.0, "grounded_final_accuracy": 41.66666666666667, "grounded_checkpoint_accuracy": 55.0, "grounded_hypothesis_updates": 33.19444444444444, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 75.0, "scope_exact": 58.333333333333336, "ioc_precision": 5.0, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 6, "sufficient_evidence_acquired": 58.333333333333336, "acquisition_milestone_recall": 79.16666666666666, "mean_query_calls": 27.25, "mean_returned_records": 56.333333333333336, "mean_latency_seconds": 147.00947528123893, "cost_usd": 24.276456, "cost_upper_bound_usd": 25.0621584, "mean_cost_usd": 2.023038, "mean_cost_upper_bound_usd": 2.0885132, "precision_cases": 8, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 91.66666666666667, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 33.333333333333336, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 72.22222222222221, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 36.111111111111114, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 58.333333333333336, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 34.72222222222222, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 12.962962962962962, "applicable_cases": 36 }, { "id": "host_scope", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 30.62610229276896, "applicable_cases": 27 }, { "id": "ioc_grounded_recall", "score": 96.875, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 29.166666666666668, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 100.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 41.666666666666664, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 79.16666666666667, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 58.333333333333336, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 55.0, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 33.19444444444444, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 14.699074074074073, "applicable_cases": 12 }, { "id": "host_scope", "score": 58.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 5.0, "applicable_cases": 8 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 20.0, "host_scope": 0.0, "ioc_precision": 20.0, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 60.0, "hypothesis_updates": 0.0, "factual_reconstruction": 37.5, "host_scope": 100.0, "ioc_precision": 20.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 60.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 22.22222222222222, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 40.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 90.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 30.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "anthropic/claude-sonnet-5", "label": "Anthropic: Claude Sonnet 5", "provider": "anthropic", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 58.333333333333336, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 20.37037037037037, "grounded_hypothesis_updates": 15.740740740740742, "workflow_completion": 100.0, "scope_precision": 97.22222222222221, "scope_recall": 100.0, "scope_exact": 94.44444444444444, "ioc_precision": 14.506172839506174, "ioc_recall": 40.625, "ioc_grounded_recall": 40.625, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 8.333333333333332, "false_separate_findings": 11, "sufficient_evidence_acquired": 2.7777777777777777, "acquisition_milestone_recall": 37.03703703703704, "mean_query_calls": 12.0, "mean_returned_records": 14.083333333333334, "mean_latency_seconds": 142.43941258799492, "cost_usd": 20.83886, "cost_upper_bound_usd": 20.83886, "mean_cost_usd": 0.5788572222222222, "mean_cost_upper_bound_usd": 0.5788572222222222, "precision_cases": 27, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 66.66666666666666, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 12.5, "grounded_hypothesis_updates": 10.13888888888889, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 55.55555555555556, "scope_exact": 33.33333333333333, "ioc_precision": 5.833333333333333, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 0.0, "false_separate_findings": 6, "sufficient_evidence_acquired": 16.666666666666664, "acquisition_milestone_recall": 56.111111111111114, "mean_query_calls": 29.166666666666668, "mean_returned_records": 51.416666666666664, "mean_latency_seconds": 234.3288487979929, "cost_usd": 16.295634, "cost_upper_bound_usd": 16.295634, "mean_cost_usd": 1.3579695, "mean_cost_upper_bound_usd": 1.3579695, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 58.333333333333336, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 37.03703703703703, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 20.37037037037037, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 15.740740740740742, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 5.5555555555555545, "applicable_cases": 36 }, { "id": "host_scope", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 14.506172839506172, "applicable_cases": 27 }, { "id": "ioc_grounded_recall", "score": 40.625, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 8.333333333333334, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 56.111111111111114, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 12.5, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 10.13888888888889, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 8.148148148148147, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 5.833333333333333, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 0.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 25.0, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 11.11111111111111, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "qwen/qwen3.8-flash", "label": "Qwen: Qwen3.8 Flash", "provider": "alibaba", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 75.0, "grounded_final_accuracy": 2.7777777777777777, "grounded_checkpoint_accuracy": 29.629629629629633, "grounded_hypothesis_updates": 21.296296296296298, "workflow_completion": 100.0, "scope_precision": 98.61111111111111, "scope_recall": 100.0, "scope_exact": 97.22222222222221, "ioc_precision": 16.129032258064516, "ioc_recall": 59.375, "ioc_grounded_recall": 59.375, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 62.5, "false_separate_findings": 8, "sufficient_evidence_acquired": 2.7777777777777777, "acquisition_milestone_recall": 46.2962962962963, "mean_query_calls": 11.944444444444445, "mean_returned_records": 33.166666666666664, "mean_latency_seconds": 356.9660487373395, "cost_usd": 0.78992872, "cost_upper_bound_usd": 1.026260608, "mean_cost_usd": 0.021942464444444445, "mean_cost_upper_bound_usd": 0.028507239111111114, "precision_cases": 31, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 83.33333333333334, "grounded_final_accuracy": 16.666666666666664, "grounded_checkpoint_accuracy": 35.0, "grounded_hypothesis_updates": 20.694444444444446, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 69.44444444444444, "scope_exact": 50.0, "ioc_precision": 3.75, "ioc_recall": 75.0, "ioc_grounded_recall": 75.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 6, "sufficient_evidence_acquired": 16.666666666666664, "acquisition_milestone_recall": 63.88888888888889, "mean_query_calls": 28.416666666666668, "mean_returned_records": 121.41666666666667, "mean_latency_seconds": 601.5948421422896, "cost_usd": 0.56009432, "cost_upper_bound_usd": 0.660016928, "mean_cost_usd": 0.04667452666666667, "mean_cost_upper_bound_usd": 0.05500141066666667, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 75.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 46.29629629629629, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 2.7777777777777777, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 29.629629629629633, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 21.296296296296298, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 0.9259259259259258, "applicable_cases": 36 }, { "id": "host_scope", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 16.129032258064516, "applicable_cases": 31 }, { "id": "ioc_grounded_recall", "score": 59.375, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 62.5, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 50.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 83.33333333333333, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 63.888888888888886, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 35.0, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 20.694444444444446, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 0.0, "applicable_cases": 12 }, { "id": "host_scope", "score": 50.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 3.75, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 75.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 50.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 12.5, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 40.0, "hypothesis_updates": 30.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 80.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 58.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "google/gemma-4-31b-it", "label": "Google: Gemma 4 31B", "provider": "coreweave/fp4", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 58.333333333333336, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 18.51851851851852, "grounded_hypothesis_updates": 16.203703703703706, "workflow_completion": 94.44444444444444, "scope_precision": 100.0, "scope_recall": 94.44444444444444, "scope_exact": 94.44444444444444, "ioc_precision": 29.545454545454547, "ioc_recall": 43.75, "ioc_grounded_recall": 43.75, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 16.666666666666664, "false_separate_findings": 20, "sufficient_evidence_acquired": 5.555555555555555, "acquisition_milestone_recall": 44.44444444444444, "mean_query_calls": 11.472222222222221, "mean_returned_records": 17.666666666666668, "mean_latency_seconds": 230.22195446654385, "cost_usd": 0.863964, "cost_upper_bound_usd": 0.877309872, "mean_cost_usd": 0.023999, "mean_cost_upper_bound_usd": 0.024369718666666665, "precision_cases": 22, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 34, "invalid": 2 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 50.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 7.777777777777778, "grounded_hypothesis_updates": 6.388888888888889, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 55.55555555555556, "scope_exact": 33.33333333333333, "ioc_precision": 8.333333333333332, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 8, "sufficient_evidence_acquired": 16.666666666666664, "acquisition_milestone_recall": 43.333333333333336, "mean_query_calls": 19.0, "mean_returned_records": 45.333333333333336, "mean_latency_seconds": 527.4538238333092, "cost_usd": 0.60836304, "cost_upper_bound_usd": 0.67930428, "mean_cost_usd": 0.05069692, "mean_cost_upper_bound_usd": 0.056608689999999996, "precision_cases": 10, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 58.333333333333336, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 44.444444444444436, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 5.555555555555555, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 18.51851851851852, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 16.203703703703706, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 3.2407407407407405, "applicable_cases": 36 }, { "id": "host_scope", "score": 94.44444444444444, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 29.545454545454543, "applicable_cases": 22 }, { "id": "ioc_grounded_recall", "score": 43.75, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 16.666666666666668, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 50.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 50.0, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 43.333333333333336, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 7.777777777777778, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 6.388888888888889, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 6.597222222222221, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 8.333333333333332, "applicable_cases": 10 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 83.33333333333334, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 12.5, "host_scope": 0.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-6-astra", "label": "OpenAI: GPT-6 Astra", "provider": "openai/flex", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 77.77777777777779, "grounded_final_accuracy": 69.44444444444444, "grounded_checkpoint_accuracy": 82.4074074074074, "grounded_hypothesis_updates": 50.0, "workflow_completion": 100.0, "scope_precision": 98.61111111111111, "scope_recall": 100.0, "scope_exact": 97.22222222222221, "ioc_precision": 22.085385878489326, "ioc_recall": 81.25, "ioc_grounded_recall": 81.25, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 100.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 100.0, "acquisition_milestone_recall": 100.0, "mean_query_calls": 10.38888888888889, "mean_returned_records": 20.97222222222222, "mean_latency_seconds": 98.93069325007835, "cost_usd": 8.953546, "cost_upper_bound_usd": 17.890066, "mean_cost_usd": 0.2487096111111111, "mean_cost_upper_bound_usd": 0.4969462777777778, "precision_cases": 29, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 66.66666666666666, "grounded_final_accuracy": 41.66666666666667, "grounded_checkpoint_accuracy": 72.22222222222223, "grounded_hypothesis_updates": 43.888888888888886, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 72.22222222222221, "scope_exact": 58.333333333333336, "ioc_precision": 6.115702479338843, "ioc_recall": 100.0, "ioc_grounded_recall": 100.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 100.0, "false_separate_findings": 0, "sufficient_evidence_acquired": 83.33333333333334, "acquisition_milestone_recall": 90.83333333333333, "mean_query_calls": 21.75, "mean_returned_records": 79.08333333333333, "mean_latency_seconds": 152.10258107309346, "cost_usd": 6.55238725, "cost_upper_bound_usd": 8.76604225, "mean_cost_usd": 0.5460322708333333, "mean_cost_upper_bound_usd": 0.7305035208333334, "precision_cases": 11, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 77.77777777777777, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 69.44444444444444, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 100.0, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 100.0, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 82.4074074074074, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 50.0, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 16.203703703703702, "applicable_cases": 36 }, { "id": "host_scope", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 22.085385878489326, "applicable_cases": 29 }, { "id": "ioc_grounded_recall", "score": 81.25, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 100.0, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 28.57142857142857, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 41.666666666666664, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 90.83333333333333, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 83.33333333333333, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 72.22222222222223, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 43.888888888888886, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 28.194444444444443, "applicable_cases": 12 }, { "id": "host_scope", "score": 58.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 6.115702479338843, "applicable_cases": 11 }, { "id": "ioc_grounded_recall", "score": 100.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 100.0, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 50.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 40.0, "host_scope": 100.0, "ioc_precision": 27.27272727272727, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 0.0, "factual_reconstruction": 25.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 80.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 40.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 80.0, "hypothesis_updates": 20.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 40.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "openai/gpt-5.6-sol", "label": "OpenAI: GPT-5.6 Sol", "provider": "openai/flex", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 80.55555555555556, "grounded_final_accuracy": 19.444444444444446, "grounded_checkpoint_accuracy": 50.0, "grounded_hypothesis_updates": 41.66666666666667, "workflow_completion": 100.0, "scope_precision": 98.61111111111111, "scope_recall": 100.0, "scope_exact": 97.22222222222221, "ioc_precision": 30.80246913580247, "ioc_recall": 81.25, "ioc_grounded_recall": 81.25, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 16.666666666666664, "false_separate_findings": 9, "sufficient_evidence_acquired": 19.444444444444446, "acquisition_milestone_recall": 56.481481481481474, "mean_query_calls": 12.0, "mean_returned_records": 13.916666666666666, "mean_latency_seconds": 49.31569668425558, "cost_usd": 1.7642888, "cost_upper_bound_usd": 1.7642888, "mean_cost_usd": 0.049008022222222225, "mean_cost_upper_bound_usd": 0.049008022222222225, "precision_cases": 27, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "completed": 36 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 91.66666666666666, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 33.88888888888889, "grounded_hypothesis_updates": 23.333333333333336, "workflow_completion": 100.0, "scope_precision": 100.0, "scope_recall": 58.333333333333336, "scope_exact": 33.33333333333333, "ioc_precision": 9.375, "ioc_recall": 41.666666666666664, "ioc_grounded_recall": 41.666666666666664, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 42.857142857142854, "false_separate_findings": 3, "sufficient_evidence_acquired": 0.0, "acquisition_milestone_recall": 47.22222222222222, "mean_query_calls": 30.0, "mean_returned_records": 59.0, "mean_latency_seconds": 72.76881959019617, "cost_usd": 1.1456197, "cost_upper_bound_usd": 1.1456197, "mean_cost_usd": 0.09546830833333332, "mean_cost_upper_bound_usd": 0.09546830833333332, "precision_cases": 8, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "completed": 12 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 80.55555555555556, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 19.444444444444443, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 56.481481481481474, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 19.444444444444443, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 50.0, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 41.66666666666667, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 19.444444444444443, "applicable_cases": 36 }, { "id": "host_scope", "score": 97.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 30.80246913580247, "applicable_cases": 27 }, { "id": "ioc_grounded_recall", "score": 81.25, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 16.666666666666668, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 40.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 50.0, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 50.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 66.66666666666667, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 50.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 91.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 47.22222222222222, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 0.0, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 33.88888888888889, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 23.333333333333336, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 26.388888888888886, "applicable_cases": 12 }, { "id": "host_scope", "score": 33.333333333333336, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 9.375, "applicable_cases": 8 }, { "id": "ioc_grounded_recall", "score": 41.666666666666664, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 42.857142857142854, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 40.0, "host_scope": 0.0, "ioc_precision": 25.0, "ioc_grounded_recall": 33.33333333333333, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": true, "evidence_backed": true }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": true, "evidence_backed": true }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": true, "evidence_backed": true }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 8.333333333333334, "factual_reconstruction": 66.66666666666666, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": true, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 80.0, "hypothesis_updates": 50.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 20.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 10.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 80.0, "hypothesis_updates": 80.0, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 16.666666666666668, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 50.0, "sufficient_evidence": 0.0, "stage_decisions": 50.0, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] }, { "model_id": "qwen/qwen3.8-27b", "label": "Qwen: Qwen3.8 27B", "provider": "deepinfra/bf16", "effort": "high", "cases": 48, "repeats": 1, "max_output_tokens": 16384, "tiers": [ { "tier": "basic", "cases": 36, "final_verdict_accuracy": 50.0, "grounded_final_accuracy": 0.0, "grounded_checkpoint_accuracy": 23.14814814814815, "grounded_hypothesis_updates": 18.51851851851852, "workflow_completion": 72.22222222222221, "scope_precision": 100.0, "scope_recall": 72.22222222222221, "scope_exact": 72.22222222222221, "ioc_precision": 20.833333333333336, "ioc_recall": 43.75, "ioc_grounded_recall": 43.75, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 12.5, "false_separate_findings": 6, "sufficient_evidence_acquired": 5.555555555555555, "acquisition_milestone_recall": 40.74074074074074, "mean_query_calls": 11.972222222222221, "mean_returned_records": 15.194444444444445, "mean_latency_seconds": 849.079444825245, "cost_usd": 2.7103965, "cost_upper_bound_usd": 2.78871504, "mean_cost_usd": 0.07528879166666666, "mean_cost_upper_bound_usd": 0.07746430666666666, "precision_cases": 20, "ioc_recall_cases": 16, "environment_cases": 24, "outcome_counts": { "context_limit": 8, "completed": 26, "invalid": 2 } }, { "tier": "intrusion", "cases": 12, "final_verdict_accuracy": 66.66666666666666, "grounded_final_accuracy": 16.666666666666664, "grounded_checkpoint_accuracy": 25.555555555555557, "grounded_hypothesis_updates": 14.444444444444445, "workflow_completion": 66.66666666666666, "scope_precision": 100.0, "scope_recall": 38.88888888888889, "scope_exact": 25.0, "ioc_precision": 12.5, "ioc_recall": 25.0, "ioc_grounded_recall": 25.0, "benign_lookalikes_flagged": 0, "environment_grounded_recall": 14.285714285714285, "false_separate_findings": 2, "sufficient_evidence_acquired": 16.666666666666664, "acquisition_milestone_recall": 53.61111111111111, "mean_query_calls": 25.416666666666668, "mean_returned_records": 50.083333333333336, "mean_latency_seconds": 1626.3220243194664, "cost_usd": 1.447164675, "cost_upper_bound_usd": 1.507082895, "mean_cost_usd": 0.12059705625, "mean_cost_upper_bound_usd": 0.12559024124999998, "precision_cases": 4, "ioc_recall_cases": 2, "environment_cases": 7, "outcome_counts": { "context_limit": 4, "completed": 8 } } ], "breakdowns": [ { "tier": "basic", "tasks": [ { "id": "final_decision", "score": 50.0, "applicable_cases": 36 }, { "id": "decision_with_proof", "score": 0.0, "applicable_cases": 36 }, { "id": "evidence_milestones", "score": 40.74074074074073, "applicable_cases": 36 }, { "id": "sufficient_evidence", "score": 5.555555555555555, "applicable_cases": 36 }, { "id": "stage_decisions", "score": 23.14814814814815, "applicable_cases": 36 }, { "id": "hypothesis_updates", "score": 18.51851851851852, "applicable_cases": 36 }, { "id": "factual_reconstruction", "score": 3.7037037037037033, "applicable_cases": 36 }, { "id": "host_scope", "score": 72.22222222222223, "applicable_cases": 36 }, { "id": "ioc_precision", "score": 20.833333333333332, "applicable_cases": 20 }, { "id": "ioc_grounded_recall", "score": 43.75, "applicable_cases": 16 }, { "id": "separate_incidents", "score": 12.5, "applicable_cases": 24 } ], "tests": [ { "case_id": "DFIR-DEV-301", "title": "Remote management investigation", "family": "remote_management", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": true, "evidence_backed": true } ] }, { "case_id": "DFIR-DEV-302", "title": "Remote management investigation", "family": "remote_management", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-303", "title": "Remote management investigation", "family": "remote_management", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-304", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-305", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-306", "title": "Scripted collection investigation", "family": "scripted_collection", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-307", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 66.66666666666666, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-308", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 33.33333333333333, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": true, "evidence_backed": true }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-309", "title": "Service lateral investigation", "family": "service_lateral", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-310", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-311", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "invalid", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-312", "title": "Wmi execution investigation", "family": "wmi_execution", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-313", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-314", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-315", "title": "Scheduled persistence investigation", "family": "scheduled_persistence", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-316", "title": "Webshell investigation", "family": "webshell", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-317", "title": "Webshell investigation", "family": "webshell", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-318", "title": "Webshell investigation", "family": "webshell", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-319", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-320", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-321", "title": "Browser credentials investigation", "family": "browser_credentials", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-322", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 33.33333333333333, "ioc_grounded_recall": 50.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-323", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-324", "title": "Data transfer investigation", "family": "data_transfer", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 50.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-325", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-326", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-327", "title": "Mailbox bec investigation", "family": "mailbox_bec", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-328", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-329", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-330", "title": "Cloud app abuse investigation", "family": "cloud_app_abuse", "outcome": "completed", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-331", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-332", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-333", "title": "Unwanted extension investigation", "family": "unwanted_extension", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 100.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-334", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 25.0, "ioc_grounded_recall": 100.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-335", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-336", "title": "Installer sideload investigation", "family": "installer_sideload", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 33.333333333333336, "hypothesis_updates": 33.333333333333336, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-010", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-011", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-012", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] }, { "tier": "intrusion", "tasks": [ { "id": "final_decision", "score": 66.66666666666667, "applicable_cases": 12 }, { "id": "decision_with_proof", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "evidence_milestones", "score": 53.61111111111111, "applicable_cases": 12 }, { "id": "sufficient_evidence", "score": 16.666666666666668, "applicable_cases": 12 }, { "id": "stage_decisions", "score": 25.555555555555557, "applicable_cases": 12 }, { "id": "hypothesis_updates", "score": 14.444444444444445, "applicable_cases": 12 }, { "id": "factual_reconstruction", "score": 4.537037037037036, "applicable_cases": 12 }, { "id": "host_scope", "score": 25.0, "applicable_cases": 12 }, { "id": "ioc_precision", "score": 12.5, "applicable_cases": 4 }, { "id": "ioc_grounded_recall", "score": 25.0, "applicable_cases": 2 }, { "id": "separate_incidents", "score": 14.285714285714286, "applicable_cases": 7 } ], "tests": [ { "case_id": "DFIR-DEV-337", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": 0.0, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-338", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-339", "title": "Loader to ransomware investigation", "family": "loader_to_ransomware", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 33.33333333333333, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-340", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": 50.0, "ioc_grounded_recall": 50.0, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "exfiltration_destination", "question": "Subject: incident. Value: domain that received stolen data.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-341", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-342", "title": "Exposed web app investigation", "family": "exposed_web_app", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 100.0, "evidence_milestones": 100.0, "sufficient_evidence": 100.0, "stage_decisions": 100.0, "hypothesis_updates": 100.0, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-343", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 60.0, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 11.11111111111111, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-019", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": true, "evidence_backed": true }, { "id": "C-021", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-344", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 40.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": null }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-345", "title": "Fake browser update investigation", "family": "fake_browser_update", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 80.0, "sufficient_evidence": 0.0, "stage_decisions": 20.0, "hypothesis_updates": 20.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-016", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-017", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-018", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-346", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 16.666666666666664, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 10.0, "host_scope": 0.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "source_address", "question": "Subject: host or tenant. Value: IPv4 address the alerted activity originated from.", "correct": true, "evidence_backed": true }, { "id": "C-020", "task": "source_host", "question": "Subject: host. Value: hostname that initiated the alerted remote activity.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-022", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false }, { "id": "C-023", "task": "entry_method", "question": "Subject: incident. Value: observed entry artifact (URL, image path or source IP), with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-024", "task": "patient_zero", "question": "Subject: incident. Value: first host where attacker code ran.", "correct": false, "evidence_backed": false }, { "id": "C-025", "task": "first_observed_execution", "question": "Subject: host. Value: image path of the first attacker process, with observed_at.", "correct": false, "evidence_backed": false }, { "id": "C-026", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-027", "task": "remote_execution_target", "question": "Subject: source host. Value: host reached by lateral movement, with observed_at of the first execution there.", "correct": false, "evidence_backed": false }, { "id": "C-028", "task": "impact_start", "question": "Subject: incident. Value: host where destructive impact began, with observed_at.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-347", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "context_limit", "task_scores": { "final_decision": 0.0, "decision_with_proof": 0.0, "evidence_milestones": 33.33333333333333, "sufficient_evidence": 0.0, "stage_decisions": 0.0, "hypothesis_updates": 0.0, "factual_reconstruction": 0.0, "host_scope": 0.0, "ioc_precision": null, "ioc_grounded_recall": null, "separate_incidents": 0.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] }, { "case_id": "DFIR-DEV-348", "title": "External rdp bruteforce investigation", "family": "external_rdp_bruteforce", "outcome": "completed", "task_scores": { "final_decision": 100.0, "decision_with_proof": 0.0, "evidence_milestones": 66.66666666666666, "sufficient_evidence": 0.0, "stage_decisions": 66.66666666666667, "hypothesis_updates": 16.666666666666668, "factual_reconstruction": 0.0, "host_scope": 100.0, "ioc_precision": 0.0, "ioc_grounded_recall": null, "separate_incidents": 100.0 }, "finding_checks": [ { "id": "C-019", "task": "initiating_process", "question": "Subject: host. Value: full image path of the process that started the alerted activity.", "correct": false, "evidence_backed": false }, { "id": "C-020", "task": "collection_limit", "question": "Subject: host or tenant. Value: collection channel name; observed_at is the last verified collection boundary for that channel.", "correct": false, "evidence_backed": false }, { "id": "C-021", "task": "operation", "question": "Subject: host or tenant. Value: the exact observed command, task action or API operation that explains the activity.", "correct": false, "evidence_backed": false } ] } ] } ] } ], "metrics": { "final_verdict_accuracy": "Correct final decision", "grounded_final_accuracy": "Final decision with proof", "scope_precision": "Correctly scoped hosts / reported hosts", "scope_recall": "Found scoped hosts / expected hosts", "ioc_precision": "Correct indicators / reported indicators", "ioc_recall": "Found indicators / expected indicators", "ioc_grounded_recall": "Indicators found with accepted proof / expected indicators", "benign_lookalikes_flagged": "Non-compromise artifacts incorrectly called indicators", "environment_grounded_recall": "Separate incidents found with proof / expected separate incidents", "false_separate_findings": "Independent incidents reported without a matching evidence key", "mean_latency_seconds": "Mean recorded time per case", "mean_cost_usd": "Mean recorded cost per case", "grounded_checkpoint_accuracy": "Correct evidence-backed decisions at sealed stages, averaged equally across cases", "grounded_hypothesis_updates": "Correct evidence-backed malicious and benign hypothesis updates", "workflow_completion": "Completed case workflows / all case attempts", "sufficient_evidence_acquired": "Cases where queries acquired the required proof / all case attempts" }, "release_hash": "5e38e2573494a3ce7136d2c680c7dfd55d10d55834c6c8d5883e189b91abcb33", "campaign_id": "dfir06-full-001", "campaign_hash": "40bb6b55cc82d85cd93ec937e86b41c07670ab143355178f22d58b8aa0be69c8", "generated_at": "2026-09-30T15:23:53.795252Z", "source_commits": [ "61d32620dd49784f60d27032caf371a0cb4f8bbc" ], "tactical_tasks": [ { "id": "final_decision", "label": "Reach the right decision", "description": "Correct final verdict, without requiring citations." }, { "id": "decision_with_proof", "label": "Support the final decision", "description": "Correct final verdict with an accepted set of evidence the agent retrieved." }, { "id": "evidence_milestones", "label": "Find evidence along the way", "description": "Fraction of required evidence milestones reached through tool queries." }, { "id": "sufficient_evidence", "label": "Gather sufficient evidence", "description": "Cases where queries acquired a complete accepted proof set, regardless of the final verdict." }, { "id": "stage_decisions", "label": "Assess evidence at each stage", "description": "Correct evidence-backed decisions at sealed checkpoints. Missing decisions receive zero." }, { "id": "hypothesis_updates", "label": "Test competing explanations", "description": "Correct evidence-backed updates to both malicious and benign hypotheses. Missing updates receive zero." }, { "id": "factual_reconstruction", "label": "Reconstruct the facts", "description": "Required factual findings correctly reported with accepted supporting evidence." }, { "id": "host_scope", "label": "Identify the exact host scope", "description": "Cases where the reported host set exactly matches the expected scope." }, { "id": "ioc_precision", "label": "Attribute indicators precisely", "description": "Correct indicators divided by reported indicators, averaged across cases reporting indicators. No reported indicators is not applicable." }, { "id": "ioc_grounded_recall", "label": "Find indicators with proof", "description": "Expected indicators found with accepted evidence, averaged across cases with expected indicators." }, { "id": "separate_incidents", "label": "Recognize separate incidents", "description": "Evidence-backed recall of other incidents, averaged across cases containing a separate incident." } ] }