completed openai batch work

analysis/jobs/f452-1/prompt.txt

@@ -0,0 +1,23 @@
+You are an expert policy analyst classifying public comments submitted to the Virginia Town Hall
+regulatory comment system. You will be given the text of a proposed regulation and a single
+public comment. Return ONLY a JSON object — no other text.
+
+Definitions:
+- stance: the commenter's position on whether the regulation should be adopted.
+  "support" = wants it approved (as-is or with changes);
+  "oppose"  = wants it rejected or substantially weakened;
+  "neutral" = takes no position, asks a question, or provides factual input only;
+  "unknown" = too vague, off-topic, or uninterpretable to classify.
+- tone: the emotional register of the writing, independent of stance.
+  "positive" = affirming, hopeful, appreciative;
+  "negative" = angry, fearful, alarmed, or contemptuous;
+  "neutral"  = matter-of-fact, procedural, or informational;
+  "mixed"    = contains both positive and negative emotional content;
+  "unclear"  = tone cannot be determined (e.g., a one-word comment).
+- stance_confidence: float 0.0-1.0, your confidence in the stance label.
+- stance_rationale: 1-3 sentences explaining the key evidence; quote specific phrases where possible.
+- tags: up to 5 short topic labels relevant to the comment's specific concerns (e.g.
+  "parental rights", "student safety", "privacy", "religious freedom", "LGBTQ+ inclusion",
+  "bullying prevention", "school sports", "bathroom access"). Empty array if none apply.
+
+Return exactly these keys: stance, stance_confidence, stance_rationale, tone, tags.

analysis/jobs/f452-1/report.json

@@ -0,0 +1,43 @@
+{
+  "prompt": "analysis\\prompt-1.txt",
+  "prompt_hash": "cb41250",
+  "input_file": "output\\f452.jsonl",
+  "input_sha256": "59dcc8b13cc2a386977a8b934c498c7e639b7e684a94ca1bfd10a14878670018",
+  "total_comments": 9083,
+  "input_tokens": 6397254,
+  "gpt-5.5": {
+    "jobs": 9,
+    "cost_$": 15.9931,
+    "est_queue_days": 7.11
+  },
+  "gpt-5.4": {
+    "jobs": 9,
+    "cost_$": 7.9966,
+    "est_queue_days": 7.11
+  },
+  "gpt-5.4-mini": {
+    "jobs": 4,
+    "cost_$": 2.399,
+    "est_queue_days": 3.2
+  },
+  "gpt-5.4-nano": {
+    "jobs": 40,
+    "cost_$": 0.6397,
+    "est_queue_days": 31.99
+  },
+  "gpt-4o": {
+    "jobs": 9,
+    "cost_$": 7.9966,
+    "est_queue_days": 7.11
+  },
+  "gpt-4o-mini": {
+    "jobs": 4,
+    "cost_$": 0.4798,
+    "est_queue_days": 3.2
+  },
+  "gpt-o4-mini": {
+    "jobs": 4,
+    "cost_$": 3.5185,
+    "est_queue_days": 3.2
+  }
+}

analysis/jobs/f452-1/status.json

@@ -0,0 +1,57 @@
+{
+  "model": "gpt-5.4-mini",
+  "prompt_hash": "cb41250",
+  "input_file": "output\\f452.jsonl",
+  "input_sha256": "59dcc8b13cc2a386977a8b934c498c7e639b7e684a94ca1bfd10a14878670018",
+  "total_comments": 9083,
+  "input_tokens": 6397254,
+  "est_queue_days": 3.2,
+  "cost_$": 2.399,
+  "total_jobs": 4,
+  "jobs": [
+    {
+      "job_num": 1,
+      "run_id": "76c97113-63aa-43db-8f84-9c60ebcbb105",
+      "status": "completed",
+      "batch_id": "batch_69fb9081639881909be0c40d86edd747",
+      "records_submitted": 2270,
+      "records_completed": 2270,
+      "records_failed": 0,
+      "submitted_at": "2026-05-06T19:03:28.949240+00:00",
+      "completed_at": "2026-05-06T20:09:14+00:00"
+    },
+    {
+      "job_num": 2,
+      "run_id": "b8f3b0bb-f155-4a5c-acce-f3504c0e09aa",
+      "status": "completed",
+      "batch_id": "batch_69fba02df7b481909e96afa1ee8879f5",
+      "records_submitted": 2274,
+      "records_completed": 2274,
+      "records_failed": 0,
+      "submitted_at": "2026-05-06T20:10:21.424330+00:00",
+      "completed_at": "2026-05-06T20:37:11+00:00"
+    },
+    {
+      "job_num": 3,
+      "run_id": "8d769f37-6beb-4a1b-87ee-3f66cdc6adc8",
+      "status": "completed",
+      "batch_id": "batch_69fba69a85488190977792b6f95b614b",
+      "records_submitted": 2282,
+      "records_completed": 2282,
+      "records_failed": 0,
+      "submitted_at": "2026-05-06T20:37:45.586815+00:00",
+      "completed_at": "2026-05-06T21:09:24+00:00"
+    },
+    {
+      "job_num": 4,
+      "run_id": "e6affbc2-ddc9-43a6-b8e9-d1f47e736283",
+      "status": "completed",
+      "batch_id": "batch_69fbe44565748190ad19f17ee3143f8d",
+      "records_submitted": 2257,
+      "records_completed": 2257,
+      "records_failed": 0,
+      "submitted_at": "2026-05-07T01:00:52.886953+00:00",
+      "completed_at": "2026-05-07T09:20:01+00:00"
+    }
+  ]
+}

analysis/openai_batch.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-analysis_batch.py — OpenAI Batch API job runner
+openai_batch.py — OpenAI Batch API job runner
 
 Run tokenizer.py first to generate report.json, then:
     create   <report.json> --model <model>   — build job directory
@@ -8,7 +8,7 @@ Run tokenizer.py first to generate report.json, then:
     status   [--job N] [--dir DIR]           — check job status
     download [--job N] [--dir DIR]           — download + normalize completed jobs
 
-DIR is a name under analysis/gpt4o/jobs/ (default: most recently created).
+DIR is a name under analysis/jobs/ (default: most recently created).
 """
 
 import argparse
@@ -52,17 +52,24 @@ _MODEL_ENCODING: dict[str, str] = {
     "gpt-4o-mini":   "o200k_base",
     "gpt-o4-mini":   "o200k_base",
 }
-_LIMIT_BUFFER = 0.90
+_LIMIT_BUFFER = 0.80
 
 
 def estimate_tokens(messages: list[dict], model: str) -> int:
-    """Exact token count via tiktoken; falls back to chars/3 + 4 overhead per message."""
+    """Token count per OpenAI cookbook chat formula; falls back to chars/3."""
     try:
         import tiktoken
         enc = tiktoken.get_encoding(_MODEL_ENCODING.get(model, "o200k_base"))
-        return sum(4 + len(enc.encode(m["content"])) for m in messages)
+        # Per OpenAI cookbook for gpt-4o: 3 overhead per message + role + content;
+        # plus 3 tokens for the reply primer (<|start|>assistant<|message|>).
+        total = 3  # reply primer
+        for m in messages:
+            total += 3
+            total += len(enc.encode(m.get("role", "")))
+            total += len(enc.encode(m["content"]))
+        return total
     except ImportError:
-        return sum(4 + len(m["content"]) // 3 for m in messages)
+        return 3 + sum(3 + len(m["content"]) // 3 for m in messages)
 
 
 def chunk_comments_by_tokens(
@@ -91,7 +98,7 @@ def chunk_comments_by_tokens(
 # ---------------------------------------------------------------------------
 # Prompt
 
-_DEFAULT_PROMPT_FILE = Path(__file__).parent.parent / "prompt-1.txt"
+_DEFAULT_PROMPT_FILE = Path(__file__).parent / "prompt-1.txt"
 SYSTEM_PROMPT = _DEFAULT_PROMPT_FILE.read_text(encoding="utf-8").strip()
 PROMPT_VERSION = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:7]
 
@@ -375,7 +382,7 @@ def cmd_create(args) -> None:
 
     print(f"Created: {job_dir.name}")
     print(f"  {len(chunks)} job(s)  |  {len(comments)} comments  |  model: {args.model}")
-    print(f"\nNext:  python analysis/gpt4o/analysis_batch.py submit")
+    print(f"\nNext:  python analysis/openai_batch.py submit")
 
 
 # ---------------------------------------------------------------------------
@@ -431,7 +438,7 @@ def cmd_submit(args, client) -> None:
     save_status(status, job_dir)
 
     print(f"Job {n} submitted: {batch.id}  ({batch.status})")
-    print(f"  python analysis/gpt4o/analysis_batch.py status")
+    print(f"  python analysis/openai_batch.py status")
 
 
 # ---------------------------------------------------------------------------

analysis/openai_realtime.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 """
-analysis/gpt4o/analysis-realtime.py — Synchronous GPT-4o pipeline for VA Townhall comments.
+analysis/openai_realtime.py — Synchronous GPT-4o pipeline for VA Townhall comments.
 
 Usage:
-    python analysis/gpt4o/analysis-realtime.py <input_jsonl> [--limit {5,10,20,50}] [--model MODEL]
+    python analysis/openai_realtime.py <input_jsonl> [--limit {5,10,20,50}] [--model MODEL]
 
 Output:
-    analysis/gpt4o/forum{id}_{scrape_ts}_{model}_{run_ts}.jsonl
+    analysis/forum{id}_{scrape_ts}_{model}_{run_ts}.jsonl
 """
 
 import argparse
@@ -30,7 +30,7 @@ except ImportError:
 # ---------------------------------------------------------------------------
 # Prompt — loaded from analysis/prompt-1.txt at import time
 
-_PROMPT_FILE = Path(__file__).parent.parent / "prompt-1.txt"
+_PROMPT_FILE = Path(__file__).parent / "prompt-1.txt"
 SYSTEM_PROMPT = _PROMPT_FILE.read_text(encoding="utf-8").strip()
 PROMPT_VERSION = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:7]
 

analysis/tokenizer.py

@@ -3,10 +3,11 @@
 tokenizer.py — estimate token usage and cost for a batch analysis run.
 
 Usage:
-    python analysis/gpt4o/tokenizer.py output/f452.jsonl [--prompt analysis/prompt-1.txt]
+    python analysis/tokenizer.py output/f452.jsonl [--prompt analysis/prompt-1.txt]
+    python analysis/tokenizer.py analysis/jobs/f452-1/job1-input.jsonl  # count actual tokens in a job
 
-Prints a per-model comparison table and writes report.json next to the input file.
-Run this before analysis_batch.py create.
+Prints a per-model comparison table and writes reports/<stem>-report.json.
+Run this before openai_batch.py create.
 """
 
 import argparse
@@ -17,7 +18,7 @@ import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent))
-import analysis_batch as _ab
+import openai_batch as _ab
 
 # Input pricing ($/1M tokens, batch API) — from docs/openai.md, updated 2026-05-05.
 # Add Anthropic/other models here when needed; only models with a LIMITS entry are reported.
@@ -66,6 +67,32 @@ def compute_report(
     return report
 
 
+def count_input_tokens(path: Path, model: str = "gpt-4o") -> dict:
+    """Count tokens in an existing job input JSONL (batch request format).
+
+    Each line must have body.messages (as written by build_batch_request_line).
+    Returns {"total_tokens": int, "total_requests": int, "min": int, "max": int, "mean": float}.
+    """
+    counts = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            req = json.loads(line)
+            messages = req["body"]["messages"]
+            counts.append(_ab.estimate_tokens(messages, model))
+    if not counts:
+        return {"total_tokens": 0, "total_requests": 0, "min": 0, "max": 0, "mean": 0.0}
+    return {
+        "total_tokens": sum(counts),
+        "total_requests": len(counts),
+        "min": min(counts),
+        "max": max(counts),
+        "mean": round(sum(counts) / len(counts), 1),
+    }
+
+
 def print_table(report: dict) -> None:
     """Print a human-readable model comparison table to stdout."""
     print(f"\nInput:    {report['input_file']}")
@@ -90,11 +117,21 @@ def print_table(report: dict) -> None:
     print()
 
 
+def _is_job_input(path: Path) -> bool:
+    """Return True if this JSONL looks like a batch request file (has custom_id)."""
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                return "custom_id" in json.loads(line)
+    return False
+
+
 def main() -> None:
-    _default_prompt = Path(__file__).parent.parent / "prompt-1.txt"
+    _default_prompt = Path(__file__).parent / "prompt-1.txt"
 
     parser = argparse.ArgumentParser(description="Estimate batch token usage and cost.")
-    parser.add_argument("input", help="Scraped JSONL file")
+    parser.add_argument("input", help="Scraped JSONL or job input JSONL (jobN-input.jsonl)")
     parser.add_argument(
         "--prompt",
         default=str(_default_prompt),
@@ -106,6 +143,16 @@ def main() -> None:
     if not input_path.exists():
         sys.exit(f"File not found: {input_path}")
 
+    # --- Mode: count tokens in an existing job input file ---
+    if _is_job_input(input_path):
+        result = count_input_tokens(input_path)
+        print(f"\nJob input: {input_path.name}")
+        print(f"  Requests : {result['total_requests']:,}")
+        print(f"  Tokens   : {result['total_tokens']:,}")
+        print(f"  Per-req  : min={result['min']}  max={result['max']}  mean={result['mean']}")
+        return
+
+    # --- Mode: estimate from raw scrape file and write report.json ---
     prompt_path = Path(args.prompt)
     if not prompt_path.exists():
         sys.exit(f"Prompt file not found: {prompt_path}")
@@ -131,10 +178,12 @@ def main() -> None:
 
     print_table(report)
 
-    out_path = input_path.parent / "report.json"
+    reports_dir = Path(__file__).parent.parent / "reports"
+    reports_dir.mkdir(exist_ok=True)
+    out_path = reports_dir / f"{input_path.stem}-report.json"
     out_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
     print(f"Report written to: {out_path}")
-    print(f"\nNext:  python analysis/gpt4o/analysis_batch.py create {out_path} --model <model>")
+    print(f"\nNext:  python analysis/openai_batch.py create {out_path} --model <model>")
 
 
 if __name__ == "__main__":