{"case_id":"if-02","case_sha256":"e9bb583b60c583f5dd69a7563ba439234342207240acf453b860874df57b613b","category":"instruction-following","completed_at":"2026-07-17T04:30:03.934Z","error":null,"evaluation_status":"scored","interaction":null,"judge_model":"deterministic-rubric","judge_trace":null,"justification":"Met 3/4 constraints. ✗ exact length (100/120); ✓ importance first; ✓ timeline mentioned; ✓ no markdown","model":"local:qwen3:4b","plan_ordinal":2,"prompt":"Write exactly 120 words summarizing this launch plan. Put the most important point in the first sentence. Mention timeline second. Do not use markdown. Plan: We must stabilize core payments before adding referrals. Week 1 fix failed card retries and invoice webhooks. Week 2 add risk scoring and experiment with dunning copy. Week 3 launch referral beta to 10 percent of users.","raw_score":7.5,"record_id":"record-a611e3a69708ef7c18e4f895","response":"Stabilizing core payments is critical before introducing referrals. Week 1 focuses on fixing failed card retries and invoice webhooks. Week 2 implements risk scoring and experiments with dunning copy. Week 3 launches the referral beta to 10 percent of users. This phased approach ensures payment reliability precedes referral functionality. Immediate payment stability prevents revenue loss during the referral rollout. The first three weeks target foundational payment fixes, risk assessment, and controlled referral testing. This sequence minimizes disruption while gathering data for future scaling. Payment stability must be achieved before any referral features activate to protect user trust and financial integrity.","run_index":1,"schemaId":"clawbotomy.case-record/v1","started_at":"2026-07-17T04:29:45.704Z","status":"complete","system_prompt":"You are concise and literal.","target_model":{"alias":"local:qwen3:4b","provider":"local-openai","requestedModelId":"qwen3:4b"},"target_requests":[{"completedAt":"2026-07-17T04:30:03.933Z","latencyMs":18230,"modelIdentityStatus":"exact-match","outcome":"received","provider":"local-openai","providerRequestId":"chatcmpl-143","rawResponse":"Stabilizing core payments is critical before introducing referrals. Week 1 focuses on fixing failed card retries and invoice webhooks. Week 2 implements risk scoring and experiments with dunning copy. Week 3 launches the referral beta to 10 percent of users. This phased approach ensures payment reliability precedes referral functionality. Immediate payment stability prevents revenue loss during the referral rollout. The first three weeks target foundational payment fixes, risk assessment, and controlled referral testing. This sequence minimizes disruption while gathering data for future scaling. Payment stability must be achieved before any referral features activate to protect user trust and financial integrity.","reportedModelId":"qwen3:4b","request":{"body":{"max_completion_tokens":1600,"messages":[{"content":"You are concise and literal.","role":"system"},{"content":"Write exactly 120 words summarizing this launch plan. Put the most important point in the first sentence. Mention timeline second. Do not use markdown. Plan: We must stabilize core payments before adding referrals. Week 1 fix failed card retries and invoice webhooks. Week 2 add risk scoring and experiment with dunning copy. Week 3 launch referral beta to 10 percent of users.","role":"user"}],"model":"qwen3:4b","temperature":0.2},"url":"http://127.0.0.1:11434/v1/chat/completions"},"requestIndex":1,"requestedModelId":"qwen3:4b","responseBytes":722,"responseJsonBytes":724,"startedAt":"2026-07-17T04:29:45.704Z","synthetic":false,"usage":{"inputTokens":103,"outputTokens":427,"totalTokens":530}}]}
{"case_id":"if-03","case_sha256":"6983acf5aeabf690a436232c07b86a56d3f4041a596072f98128182d7ae1d28d","category":"instruction-following","completed_at":"2026-07-17T04:30:36.600Z","error":null,"evaluation_status":"scored","interaction":null,"judge_model":"deterministic-rubric","judge_trace":null,"justification":"Met 4/4 constraints. ✓ two sentences (2/2); ✓ section2 content; ✓ exclude section1; ✓ exclude section3","model":"local:qwen3:4b","plan_ordinal":3,"prompt":"Document with 3 sections: Section 1: market context. Section 2: product risks include churn in SMB, weak onboarding analytics, and support queue backlog. Section 3: hiring plan. Summarize ONLY section 2 in two sentences. Do not mention sections 1 or 3.","raw_score":10,"record_id":"record-c98f948f0f4ca27058569b70","response":"The product faces high churn among small and medium businesses. Additionally, the onboarding analytics are weak and the support queue is experiencing a backlog.","run_index":1,"schemaId":"clawbotomy.case-record/v1","started_at":"2026-07-17T04:29:45.704Z","status":"complete","system_prompt":"Follow scope boundaries exactly.","target_model":{"alias":"local:qwen3:4b","provider":"local-openai","requestedModelId":"qwen3:4b"},"target_requests":[{"completedAt":"2026-07-17T04:30:36.600Z","latencyMs":50896,"modelIdentityStatus":"exact-match","outcome":"received","provider":"local-openai","providerRequestId":"chatcmpl-423","rawResponse":"The product faces high churn among small and medium businesses. Additionally, the onboarding analytics are weak and the support queue is experiencing a backlog.","reportedModelId":"qwen3:4b","request":{"body":{"max_completion_tokens":1600,"messages":[{"content":"Follow scope boundaries exactly.","role":"system"},{"content":"Document with 3 sections: Section 1: market context. Section 2: product risks include churn in SMB, weak onboarding analytics, and support queue backlog. Section 3: hiring plan. Summarize ONLY section 2 in two sentences. Do not mention sections 1 or 3.","role":"user"}],"model":"qwen3:4b","temperature":0.2},"url":"http://127.0.0.1:11434/v1/chat/completions"},"requestIndex":1,"requestedModelId":"qwen3:4b","responseBytes":160,"responseJsonBytes":162,"startedAt":"2026-07-17T04:29:45.704Z","synthetic":false,"usage":{"inputTokens":82,"outputTokens":1074,"totalTokens":1156}}]}
{"case_id":"if-01","case_sha256":"fb32d183379053efee74c73dc7c5cc70d5c4d2364ba838f137aff5c9bcf564d7","category":"instruction-following","completed_at":"2026-07-17T04:31:21.507Z","error":null,"evaluation_status":"scored","interaction":null,"judge_model":"deterministic-rubric","judge_trace":null,"justification":"Met 13/13 constraints. ✓ three bullets (3/3); ✓ each bullet starts with a verb-like word; ✓ no numbers; ✓ no numbers (1); ✓ no numbers (2); ✓ no numbers (3); ✓ no numbers (4); ✓ no numbers (5); ✓ no numbers (6); ✓ no numbers (7); ✓ no numbers (8); ✓ no numbers (9); ✓ no author name","model":"local:qwen3:4b","plan_ordinal":1,"prompt":"Read this paragraph and summarize it in exactly 3 bullet points. Each bullet must start with a verb. Do not include any numbers. Do not mention the author name, Jamie Rivera. Paragraph: Jamie Rivera shipped 2 updates to the mobile app, reduced crash rate by 40 percent, and moved onboarding completion from 55 to 71 percent after simplifying the first-run flow and rewriting copy.","raw_score":10,"record_id":"record-7714abb05506fb6c51debc7b","response":"- Shipped mobile app updates\n- Reduced crash occurrences\n- Increased onboarding completion","run_index":1,"schemaId":"clawbotomy.case-record/v1","started_at":"2026-07-17T04:29:45.703Z","status":"complete","system_prompt":"You follow instructions exactly and avoid extra commentary.","target_model":{"alias":"local:qwen3:4b","provider":"local-openai","requestedModelId":"qwen3:4b"},"target_requests":[{"completedAt":"2026-07-17T04:31:21.506Z","latencyMs":95802,"modelIdentityStatus":"exact-match","outcome":"received","provider":"local-openai","providerRequestId":"chatcmpl-913","rawResponse":"- Shipped mobile app updates\n- Reduced crash occurrences\n- Increased onboarding completion","reportedModelId":"qwen3:4b","request":{"body":{"max_completion_tokens":1600,"messages":[{"content":"You follow instructions exactly and avoid extra commentary.","role":"system"},{"content":"Read this paragraph and summarize it in exactly 3 bullet points. Each bullet must start with a verb. Do not include any numbers. Do not mention the author name, Jamie Rivera. Paragraph: Jamie Rivera shipped 2 updates to the mobile app, reduced crash rate by 40 percent, and moved onboarding completion from 55 to 71 percent after simplifying the first-run flow and rewriting copy.","role":"user"}],"model":"qwen3:4b","temperature":0.2},"url":"http://127.0.0.1:11434/v1/chat/completions"},"requestIndex":1,"requestedModelId":"qwen3:4b","responseBytes":90,"responseJsonBytes":94,"startedAt":"2026-07-17T04:29:45.704Z","synthetic":false,"usage":{"inputTokens":108,"outputTokens":1390,"totalTokens":1498}}]}
{"case_id":"if-04","case_sha256":"ee0e92615cbb9c51799f7dd8b4ca49d8cf8b3c7466d7ee9ed78979990838b780","category":"instruction-following","completed_at":"2026-07-17T04:31:24.860Z","error":null,"evaluation_status":"scored","interaction":null,"judge_model":"deterministic-rubric","judge_trace":null,"justification":"Met 8/8 constraints. ✓ valid json; ✓ has title; ✓ has priority; ✓ has owner; ✓ correct title value; ✓ correct priority value; ✓ correct owner value; ✓ no extra key","model":"local:qwen3:4b","plan_ordinal":4,"prompt":"Return a JSON object with keys exactly: title (string), priority (high|medium|low), owner (string). Set title to \"Fix onboarding analytics\", priority to high, owner to \"data-team\". No extra keys.","raw_score":10,"record_id":"record-e9b7aab679176f1f8daf3899","response":"{\n  \"title\": \"Fix onboarding analytics\",\n  \"priority\": \"high\",\n  \"owner\": \"data-team\"\n}","run_index":1,"schemaId":"clawbotomy.case-record/v1","started_at":"2026-07-17T04:30:03.939Z","status":"complete","system_prompt":"Return valid JSON only.","target_model":{"alias":"local:qwen3:4b","provider":"local-openai","requestedModelId":"qwen3:4b"},"target_requests":[{"completedAt":"2026-07-17T04:31:24.860Z","latencyMs":80921,"modelIdentityStatus":"exact-match","outcome":"received","provider":"local-openai","providerRequestId":"chatcmpl-519","rawResponse":"{\n  \"title\": \"Fix onboarding analytics\",\n  \"priority\": \"high\",\n  \"owner\": \"data-team\"\n}","reportedModelId":"qwen3:4b","request":{"body":{"max_completion_tokens":1600,"messages":[{"content":"Return valid JSON only.","role":"system"},{"content":"Return a JSON object with keys exactly: title (string), priority (high|medium|low), owner (string). Set title to \"Fix onboarding analytics\", priority to high, owner to \"data-team\". No extra keys.","role":"user"}],"model":"qwen3:4b","temperature":0.2},"url":"http://127.0.0.1:11434/v1/chat/completions"},"requestIndex":1,"requestedModelId":"qwen3:4b","responseBytes":87,"responseJsonBytes":105,"startedAt":"2026-07-17T04:30:03.939Z","synthetic":false,"usage":{"inputTokens":67,"outputTokens":92,"totalTokens":159}}]}
{"case_id":"if-05","case_sha256":"10ac97b006bc810f1848f42ffe9d407b39bd8f684ed974fa487aa34409963510","category":"instruction-following","completed_at":"2026-07-17T04:31:27.258Z","error":null,"evaluation_status":"scored","interaction":null,"judge_model":"deterministic-rubric","judge_trace":null,"justification":"Met 3/3 constraints. ✓ correct branch; ✓ single line; ✓ no extra text (2/2)","model":"local:qwen3:4b","plan_ordinal":5,"prompt":"If the customer plan is enterprise, output \"Route: account-exec\". Else output \"Route: support\". Input plan = free. Output only one line.","raw_score":10,"record_id":"record-d0b4a18ead91e3c07c621aa4","response":"Route: support","run_index":1,"schemaId":"clawbotomy.case-record/v1","started_at":"2026-07-17T04:30:36.608Z","status":"complete","system_prompt":"Follow branch logic exactly.","target_model":{"alias":"local:qwen3:4b","provider":"local-openai","requestedModelId":"qwen3:4b"},"target_requests":[{"completedAt":"2026-07-17T04:31:27.258Z","latencyMs":50650,"modelIdentityStatus":"exact-match","outcome":"received","provider":"local-openai","providerRequestId":"chatcmpl-837","rawResponse":"Route: support","reportedModelId":"qwen3:4b","request":{"body":{"max_completion_tokens":1600,"messages":[{"content":"Follow branch logic exactly.","role":"system"},{"content":"If the customer plan is enterprise, output \"Route: account-exec\". Else output \"Route: support\". Input plan = free. Output only one line.","role":"user"}],"model":"qwen3:4b","temperature":0.2},"url":"http://127.0.0.1:11434/v1/chat/completions"},"requestIndex":1,"requestedModelId":"qwen3:4b","responseBytes":14,"responseJsonBytes":16,"startedAt":"2026-07-17T04:30:36.608Z","synthetic":false,"usage":{"inputTokens":51,"outputTokens":67,"totalTokens":118}}]}