add gpt4o batch analysis

.gitignore

@@ -10,6 +10,7 @@ __pycache__/
 .venv/
 venv/
 env/
+.claude/
 
 # --- emacs ---
 *~

agents.md

@@ -38,3 +38,8 @@ Description and PM notes
 - tests: 
 - date: [2026-05-05 Tue 15:00]
 ```
+
+## tests and commands
+- project dir: `%userprofile%\projects\vath\`
+- python venv: `%userprofile%\projects\vath\venv\scripts\activate`
+- pytest (inside venv): `python -m pytest tests/`

analysis/gpt4o/analysis_batch.py

@@ -0,0 +1,420 @@
+#!/usr/bin/env python3
+"""
+analysis/gpt4o/analysis-batch.py — OpenAI Batch API pipeline
+
+Commands (run manually in order):
+    submit   <input_jsonl> [--model gpt-4o]   — build request file, upload, create batch
+    status   <run_id>                          — check batch status, update manifest
+    download <run_id>                          — download + normalize output, update manifest
+
+File layout (all under analysis/gpt4o/):
+    requests/<run_id>.jsonl     — batch input sent to OpenAI
+    raw/<run_id>.jsonl          — raw batch output from OpenAI
+    runs/<run_id>.json          — run manifest
+    <run_id>_<model>.jsonl      — normalized output (same schema as realtime)
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+try:
+    import openai
+except ImportError:
+    sys.exit("openai package not installed. Run: pip install openai")
+
+# ---------------------------------------------------------------------------
+# Prompt
+
+_DEFAULT_PROMPT_FILE = Path(__file__).parent.parent / "prompt-1.txt"
+SYSTEM_PROMPT = _DEFAULT_PROMPT_FILE.read_text(encoding="utf-8").strip()
+PROMPT_VERSION = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:7]
+
+
+def _load_prompt(path: Path) -> None:
+    """Re-read a prompt file, updating module-level SYSTEM_PROMPT and PROMPT_VERSION."""
+    global SYSTEM_PROMPT, PROMPT_VERSION
+    SYSTEM_PROMPT = path.read_text(encoding="utf-8").strip()
+    PROMPT_VERSION = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:7]
+
+USER_TEMPLATE = """\
+## Proposed Regulation
+Title: {reg_title}
+Description: {reg_desc}
+
+---
+
+## Public Comment
+Comment ID: {comment_id}
+Title: {comment_title}
+Body:
+{comment_text}
+
+---
+Classify this comment per the instructions. Return only JSON.\
+"""
+
+MAX_COMMENT_CHARS = 6000
+
+# ---------------------------------------------------------------------------
+# Directories
+
+_SCRIPT_DIR  = Path(__file__).parent
+REQUESTS_DIR = _SCRIPT_DIR / "requests"
+RAW_DIR      = _SCRIPT_DIR / "raw"
+RUNS_DIR     = _SCRIPT_DIR / "runs"
+
+# ---------------------------------------------------------------------------
+# Core functions (importable for tests)
+
+
+def load_items(path: Path) -> tuple[dict | None, list[dict]]:
+    """Read a scraped JSONL file. Returns (forum_item_or_None, [comment_items])."""
+    forum = None
+    comments = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            item = json.loads(line)
+            if "comment_id" in item:
+                comments.append(item)
+            elif "reg_title" in item:
+                forum = item
+    return forum, comments
+
+
+def custom_id_from(comment_id: str) -> str:
+    return f"comment_{comment_id}"
+
+
+def parse_custom_id(custom_id: str) -> str:
+    """Return comment_id from a custom_id string."""
+    return custom_id.removeprefix("comment_")
+
+
+def build_messages(comment: dict, forum: dict | None) -> tuple[list, bool]:
+    """Build OpenAI messages for one comment. Returns (messages, truncated)."""
+    reg_title = (forum or {}).get("reg_title", "[unknown]")
+    reg_desc  = (forum or {}).get("reg_desc",  "[unknown]")
+
+    body = (comment.get("text") or "").strip()
+    truncated = False
+    if not body:
+        body = "[No body text provided]"
+    elif len(body) > MAX_COMMENT_CHARS:
+        body = body[:MAX_COMMENT_CHARS] + "... [truncated]"
+        truncated = True
+
+    user_text = USER_TEMPLATE.format(
+        reg_title=reg_title,
+        reg_desc=reg_desc,
+        comment_id=comment.get("comment_id", ""),
+        comment_title=comment.get("title", ""),
+        comment_text=body,
+    )
+
+    return [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user",   "content": user_text},
+    ], truncated
+
+
+def build_batch_request_line(comment: dict, forum: dict | None, model: str) -> dict:
+    """Build one line of the batch input JSONL."""
+    messages, _ = build_messages(comment, forum)
+    return {
+        "custom_id": custom_id_from(comment["comment_id"]),
+        "method": "POST",
+        "url": "/v1/chat/completions",
+        "body": {
+            "model": model,
+            "messages": messages,
+            "response_format": {"type": "json_object"},
+            "temperature": 0.0,
+        },
+    }
+
+
+def normalize_output_line(
+    raw_line: dict,
+    comment_lookup: dict,
+    run_id: str,
+    analyzed_at: str,
+    model: str,
+    prompt_version: str,
+) -> dict:
+    """Convert one raw batch output line into a normalized analysis record.
+
+    comment_lookup: {comment_id: CommentItem dict}
+    prompt_version: taken from the run manifest so it reflects what was submitted.
+    """
+    comment_id = parse_custom_id(raw_line.get("custom_id", ""))
+    comment = comment_lookup.get(comment_id, {})
+
+    base = {
+        "run_id":         run_id,
+        "forum_id":       comment.get("forum_id", ""),
+        "comment_id":     comment_id,
+        "analyzed_at":    analyzed_at,
+        "model":          model,
+        "prompt_version": prompt_version,
+        "input_title":    comment.get("title", ""),
+        "truncated":      len(comment.get("text") or "") > MAX_COMMENT_CHARS,
+    }
+
+    # Check for outer-level batch error (e.g. batch_expired)
+    if raw_line.get("error"):
+        err = raw_line["error"]
+        err_msg = err.get("message", str(err)) if isinstance(err, dict) else str(err)
+        return {**base, "stance": None, "stance_confidence": None,
+                "stance_rationale": None, "tone": None, "tags": None, "error": err_msg}
+
+    response = raw_line.get("response") or {}
+    if response.get("status_code") != 200:
+        return {**base, "stance": None, "stance_confidence": None,
+                "stance_rationale": None, "tone": None, "tags": None,
+                "error": f"status {response.get('status_code')}"}
+
+    try:
+        content = response["body"]["choices"][0]["message"]["content"]
+        data = json.loads(content)
+        keys = ("stance", "stance_confidence", "stance_rationale", "tone", "tags")
+        parsed = {k: data.get(k) for k in keys}
+        return {**base, **parsed, "error": None}
+    except Exception as exc:
+        return {**base, "stance": None, "stance_confidence": None,
+                "stance_rationale": None, "tone": None, "tags": None, "error": str(exc)}
+
+
+def make_manifest(
+    run_id: str,
+    input_filename: str,
+    input_sha256: str,
+    model: str,
+    batch_id: str,
+    records_submitted: int,
+    request_filename: str,
+) -> dict:
+    return {
+        "run_id":                 run_id,
+        "input_filename":         input_filename,
+        "input_sha256":           input_sha256,
+        "prompt_hash":            PROMPT_VERSION,
+        "model":                  model,
+        "batch_id":               batch_id,
+        "records_submitted":      records_submitted,
+        "records_completed":      None,
+        "records_failed":         None,
+        "request_filename":       request_filename,
+        "raw_output_filename":    None,
+        "normalized_output_filename": None,
+        "created_at":             datetime.now(timezone.utc).isoformat(),
+        "completed_at":           None,
+    }
+
+
+def load_manifest(run_id: str) -> dict:
+    path = RUNS_DIR / f"{run_id}.json"
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def save_manifest(manifest: dict) -> None:
+    RUNS_DIR.mkdir(parents=True, exist_ok=True)
+    path = RUNS_DIR / f"{manifest['run_id']}.json"
+    path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Subcommand: submit
+
+def cmd_submit(args, client) -> None:
+    _load_prompt(Path(args.prompt))
+    print(f"Prompt: {args.prompt}  (version {PROMPT_VERSION})", file=sys.stderr)
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        sys.exit(f"File not found: {input_path}")
+
+    print(f"Reading {input_path} ...", file=sys.stderr)
+    forum, comments = load_items(input_path)
+    if not comments:
+        sys.exit("No comment items found in input file.")
+    if forum is None:
+        print("Warning: no ForumItem found — regulation context will be [unknown].", file=sys.stderr)
+
+    import uuid
+    run_id = str(uuid.uuid4())
+    input_sha256 = hashlib.sha256(input_path.read_bytes()).hexdigest()
+
+    # Build batch request file
+    REQUESTS_DIR.mkdir(parents=True, exist_ok=True)
+    request_path = REQUESTS_DIR / f"{run_id}.jsonl"
+    with open(request_path, "w", encoding="utf-8") as f:
+        for comment in comments:
+            line = build_batch_request_line(comment, forum, args.model)
+            f.write(json.dumps(line, ensure_ascii=False) + "\n")
+
+    print(f"Wrote {len(comments)} requests → {request_path}", file=sys.stderr)
+
+    # Upload to OpenAI
+    print("Uploading request file ...", file=sys.stderr)
+    with open(request_path, "rb") as f:
+        uploaded = client.files.create(file=f, purpose="batch")
+    print(f"Uploaded: {uploaded.id}", file=sys.stderr)
+
+    # Create batch
+    batch = client.batches.create(
+        input_file_id=uploaded.id,
+        endpoint="/v1/chat/completions",
+        completion_window="24h",
+        metadata={"run_id": run_id, "input_filename": str(input_path)},
+    )
+    print(f"Batch created: {batch.id}  status={batch.status}", file=sys.stderr)
+
+    # Save manifest
+    manifest = make_manifest(
+        run_id=run_id,
+        input_filename=str(input_path),
+        input_sha256=input_sha256,
+        model=args.model,
+        batch_id=batch.id,
+        records_submitted=len(comments),
+        request_filename=str(request_path),
+    )
+    save_manifest(manifest)
+
+    print(f"\nrun_id: {run_id}", file=sys.stderr)
+    print(f"Check status:   python analysis/gpt4o/analysis-batch.py status {run_id}", file=sys.stderr)
+    print(run_id)  # stdout for scripting
+
+
+# ---------------------------------------------------------------------------
+# Subcommand: status
+
+def cmd_status(args, client) -> None:
+    manifest = load_manifest(args.run_id)
+    batch = client.batches.retrieve(manifest["batch_id"])
+
+    counts = batch.request_counts
+    print(f"status:     {batch.status}")
+    print(f"completed:  {counts.completed}/{counts.total}")
+    print(f"failed:     {counts.failed}")
+
+    manifest["records_completed"] = counts.completed
+    manifest["records_failed"]    = counts.failed
+    save_manifest(manifest)
+
+    if batch.status == "completed":
+        print(f"\nReady to download. Run:")
+        print(f"  python analysis/gpt4o/analysis-batch.py download {args.run_id}")
+
+
+# ---------------------------------------------------------------------------
+# Subcommand: download
+
+def cmd_download(args, client) -> None:
+    manifest = load_manifest(args.run_id)
+    batch = client.batches.retrieve(manifest["batch_id"])
+
+    if batch.status != "completed":
+        sys.exit(f"Batch not complete yet (status={batch.status}). Run 'status' to check.")
+
+    run_id    = manifest["run_id"]
+    model     = manifest["model"]
+    model_slug = model.replace("/", "-")
+
+    # Download raw output
+    RAW_DIR.mkdir(parents=True, exist_ok=True)
+    raw_path = RAW_DIR / f"{run_id}.jsonl"
+    raw_text = client.files.content(batch.output_file_id).text
+    raw_path.write_text(raw_text, encoding="utf-8")
+    print(f"Raw output → {raw_path}", file=sys.stderr)
+
+    # Build comment lookup from original input for reconciliation
+    input_path = Path(manifest["input_filename"])
+    _, comments = load_items(input_path)
+    comment_lookup = {c["comment_id"]: c for c in comments}
+
+    # Normalize
+    completed_at = datetime.now(timezone.utc).isoformat()
+    if batch.completed_at:
+        completed_at = datetime.fromtimestamp(batch.completed_at, tz=timezone.utc).isoformat()
+
+    normalized_path = _SCRIPT_DIR / f"{run_id}_{model_slug}.jsonl"
+    n_ok = n_err = 0
+    with open(normalized_path, "w", encoding="utf-8") as out:
+        for line in raw_text.splitlines():
+            if not line.strip():
+                continue
+            raw_line = json.loads(line)
+            record = normalize_output_line(raw_line, comment_lookup, run_id, completed_at, model, manifest["prompt_hash"])
+            out.write(json.dumps(record, ensure_ascii=False) + "\n")
+            if record["error"]:
+                n_err += 1
+            else:
+                n_ok += 1
+
+    print(f"Normalized → {normalized_path}  ({n_ok} ok, {n_err} errors)", file=sys.stderr)
+
+    manifest["records_completed"]         = n_ok
+    manifest["records_failed"]            = n_err
+    manifest["raw_output_filename"]       = str(raw_path)
+    manifest["normalized_output_filename"] = str(normalized_path)
+    manifest["completed_at"]              = completed_at
+    save_manifest(manifest)
+    print(f"Manifest updated → {RUNS_DIR / run_id}.json", file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# CLI
+
+def main() -> None:
+    load_dotenv()
+
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        sys.exit("OPENAI_API_KEY not set. Create a .env file or export the variable.")
+
+    parser = argparse.ArgumentParser(
+        description="Public comment batch analysis pipeline.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_submit = sub.add_parser("submit", help="Build and submit a batch job")
+    p_submit.add_argument("input", help="Path to scraped JSONL file")
+    p_submit.add_argument("--model", default="gpt-4o", help="OpenAI model (default: gpt-4o)")
+    p_submit.add_argument(
+        "--prompt",
+        default=str(_DEFAULT_PROMPT_FILE),
+        help="Path to system prompt file (default: analysis/prompt-1.txt)",
+    )
+
+    p_status = sub.add_parser("status", help="Check batch status")
+    p_status.add_argument("run_id", help="run_id from submit output")
+
+    p_download = sub.add_parser("download", help="Download and normalize completed batch")
+    p_download.add_argument("run_id", help="run_id from submit output")
+
+    args = parser.parse_args()
+    client = openai.OpenAI(api_key=api_key)
+
+    if args.command == "submit":
+        cmd_submit(args, client)
+    elif args.command == "status":
+        cmd_status(args, client)
+    elif args.command == "download":
+        cmd_download(args, client)
+
+
+if __name__ == "__main__":
+    main()

analysis/gpt4o/analysis_realtime.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 """
-analysis/gpt4o/analysis.py — Manual GPT-4o sentiment pipeline for VA Townhall comments.
+analysis/gpt4o/analysis-realtime.py — Synchronous GPT-4o pipeline for VA Townhall comments.
 
 Usage:
-    python analysis/gpt4o/analysis.py <input_jsonl> [--limit {5,10,20,50}] [--model MODEL]
+    python analysis/gpt4o/analysis-realtime.py <input_jsonl> [--limit {5,10,20,50}] [--model MODEL]
 
 Output:
     analysis/gpt4o/forum{id}_{scrape_ts}_{model}_{run_ts}.jsonl
@@ -28,33 +28,11 @@ except ImportError:
     sys.exit("openai package not installed. Run: pip install openai")
 
 # ---------------------------------------------------------------------------
-# Prompt (version is derived from the content — changing either string changes PROMPT_VERSION)
+# Prompt — loaded from analysis/prompt-1.txt at import time
 
-SYSTEM_PROMPT = """\
-You are an expert policy analyst classifying public comments submitted to the Virginia Town Hall
-regulatory comment system. You will be given the text of a proposed regulation and a single
-public comment. Return ONLY a JSON object — no other text.
-
-Definitions:
-- stance: the commenter's position on whether the regulation should be adopted.
-  "support" = wants it approved (as-is or with changes);
-  "oppose"  = wants it rejected or substantially weakened;
-  "neutral" = takes no position, asks a question, or provides factual input only;
-  "unknown" = too vague, off-topic, or uninterpretable to classify.
-- tone: the emotional register of the writing, independent of stance.
-  "positive" = affirming, hopeful, appreciative;
-  "negative" = angry, fearful, alarmed, or contemptuous;
-  "neutral"  = matter-of-fact, procedural, or informational;
-  "mixed"    = contains both positive and negative emotional content;
-  "unclear"  = tone cannot be determined (e.g., a one-word comment).
-- stance_confidence: float 0.0–1.0, your confidence in the stance label.
-- stance_rationale: 1–3 sentences explaining the key evidence; quote specific phrases where possible.
-- tags: up to 5 short topic labels relevant to the comment's specific concerns (e.g.
-  "parental rights", "student safety", "privacy", "religious freedom", "LGBTQ+ inclusion",
-  "bullying prevention", "school sports", "bathroom access"). Empty array if none apply.
-
-Return exactly these keys: stance, stance_confidence, stance_rationale, tone, tags.\
-"""
+_PROMPT_FILE = Path(__file__).parent.parent / "prompt-1.txt"
+SYSTEM_PROMPT = _PROMPT_FILE.read_text(encoding="utf-8").strip()
+PROMPT_VERSION = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:7]
 
 USER_TEMPLATE = """\
 ## Proposed Regulation
@@ -73,15 +51,11 @@ Body:
 Classify this comment per the instructions. Return only JSON.\
 """
 
-PROMPT_VERSION = hashlib.sha256(
-    (SYSTEM_PROMPT + USER_TEMPLATE).encode("utf-8")
-).hexdigest()[:7]
-
 MAX_COMMENT_CHARS = 6000
-_RETRY_DELAYS = [1.0, 2.0]  # delays before attempt 2 and 3
+_RETRY_DELAYS = [1.0, 2.0]
 
 # ---------------------------------------------------------------------------
-# Core functions (importable for tests)
+# Core functions
 
 
 def load_items(path: Path) -> tuple[dict | None, list[dict]]:
@@ -102,11 +76,7 @@ def load_items(path: Path) -> tuple[dict | None, list[dict]]:
 
 
 def build_messages(comment: dict, forum: dict | None) -> tuple[list, bool]:
-    """Build the OpenAI messages list for one comment.
-
-    Returns (messages, truncated) where truncated is True if the comment body
-    was cut to MAX_COMMENT_CHARS.
-    """
+    """Build OpenAI messages for one comment. Returns (messages, truncated)."""
     reg_title = (forum or {}).get("reg_title", "[unknown]")
     reg_desc  = (forum or {}).get("reg_desc",  "[unknown]")
 
@@ -132,8 +102,13 @@ def build_messages(comment: dict, forum: dict | None) -> tuple[list, bool]:
     ], truncated
 
 
+def parse_api_response(content: str) -> dict:
+    data = json.loads(content)
+    keys = ("stance", "stance_confidence", "stance_rationale", "tone", "tags")
+    return {k: data.get(k) for k in keys}
+
+
 def _call_api(client, messages: list, model: str) -> str:
-    """Call the OpenAI chat API with exponential-backoff retry on rate limits."""
     last_exc = None
     for delay in [0.0] + _RETRY_DELAYS:
         if delay:
@@ -151,21 +126,7 @@ def _call_api(client, messages: list, model: str) -> str:
     raise last_exc  # type: ignore[misc]
 
 
-def parse_api_response(content: str) -> dict:
-    """Parse the model's JSON response, returning only the expected keys."""
-    data = json.loads(content)
-    keys = ("stance", "stance_confidence", "stance_rationale", "tone", "tags")
-    return {k: data.get(k) for k in keys}
-
-
-def analyze_comment(
-    client,
-    comment: dict,
-    forum: dict | None,
-    run_id: str,
-    model: str,
-) -> dict:
-    """Analyze one comment and return a fully-formed output record."""
+def analyze_comment(client, comment: dict, forum: dict | None, run_id: str, model: str) -> dict:
     base = {
         "run_id":         run_id,
         "forum_id":       comment.get("forum_id", ""),
@@ -191,7 +152,6 @@ def analyze_comment(
 
 
 def _scrape_ts_from_filename(path: Path) -> str:
-    """Extract the timestamp from a scraped JSONL filename for use in the output name."""
     m = re.search(r"(\d{4}-\d{2}-\d{2}T[\d\-+:]+)", path.stem)
     return m.group(1).replace(":", "-") if m else "unknown"
 
@@ -199,13 +159,11 @@ def _scrape_ts_from_filename(path: Path) -> str:
 # ---------------------------------------------------------------------------
 # CLI
 
-
 def main() -> None:
     load_dotenv()
 
     parser = argparse.ArgumentParser(
-        description="Analyze VA Townhall public comments with GPT-4o.",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="Analyze VA Townhall public comments with GPT-4o (synchronous).",
     )
     parser.add_argument("input", help="Path to scraped JSONL file")
     parser.add_argument(
@@ -215,11 +173,7 @@ def main() -> None:
         metavar="{5,10,20,50}",
         help="Process only the first N comments (for testing). Omit to process all.",
     )
-    parser.add_argument(
-        "--model",
-        default="gpt-4o",
-        help="OpenAI model name (default: gpt-4o)",
-    )
+    parser.add_argument("--model", default="gpt-4o", help="OpenAI model (default: gpt-4o)")
     args = parser.parse_args()
 
     api_key = os.environ.get("OPENAI_API_KEY")
@@ -234,10 +188,7 @@ def main() -> None:
     forum, comments = load_items(input_path)
 
     if forum is None:
-        print(
-            "Warning: no ForumItem found in file — regulation context will be [unknown].",
-            file=sys.stderr,
-        )
+        print("Warning: no ForumItem found — regulation context will be [unknown].", file=sys.stderr)
 
     if args.limit:
         comments = comments[: args.limit]
@@ -264,16 +215,10 @@ def main() -> None:
             out.flush()
             if record["error"]:
                 n_err += 1
-                print(
-                    f"  [{i}/{total}] ERROR {comment.get('comment_id')}: {record['error']}",
-                    file=sys.stderr,
-                )
+                print(f"  [{i}/{total}] ERROR {comment.get('comment_id')}: {record['error']}", file=sys.stderr)
             else:
                 n_ok += 1
-                print(
-                    f"  [{i}/{total}] OK    {comment.get('comment_id')} → {record['stance']}",
-                    file=sys.stderr,
-                )
+                print(f"  [{i}/{total}] OK    {comment.get('comment_id')} → {record['stance']}", file=sys.stderr)
             time.sleep(0.1)
 
     print(f"\nDone. {n_ok} ok, {n_err} errors → {out_path}", file=sys.stderr)

analysis/gpt4o/forum452_unknown_gpt-4o_2026-05-05T20-44-11+00-00.jsonl

@@ -0,0 +1,10 @@
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87914", "analyzed_at": "2026-05-05T20:44:11.731054+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "Support the Model Policy Wholeheartedly", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter explicitly states, \"I support the model policy wholeheartedly,\" indicating clear support for the regulation. They also express appreciation for the policy's inclusivity and guidance, saying it is a \"first step in creating schools in Virginia that are inclusive and welcoming for transgender and non-binary students.\"", "tone": "positive", "tags": ["LGBTQ+ inclusion", "student safety", "school policy", "transgender rights", "educational support"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87915", "analyzed_at": "2026-05-05T20:44:14.418311+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "Please support this vital policy", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter explicitly states, 'I strongly support these proposals,' indicating clear approval of the regulation. They also affirm the importance of treating every student with dignity and respect, aligning with the policy's goals.", "tone": "positive", "tags": ["LGBTQ+ inclusion", "student safety", "nondiscrimination"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87916", "analyzed_at": "2026-05-05T20:44:17.820090+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "Please support this policy", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter explicitly states 'I am in full support of this policy guidance,' indicating clear support for the regulation. The phrase 'Trans rights are human rights' further reinforces their supportive stance.", "tone": "positive", "tags": ["transgender rights", "human rights"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87917", "analyzed_at": "2026-05-05T20:44:18.982080+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "Please support this policy", "stance": "support", "stance_confidence": 0.95, "stance_rationale": "The commenter explicitly states 'Please support this policy' and 'Please implement this policy,' indicating a clear support for the adoption of the regulation.", "tone": "positive", "tags": ["transgender rights", "student safety", "nondiscrimination"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87918", "analyzed_at": "2026-05-05T20:44:22.439016+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "An Essential Policy", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter explicitly states 'I fully support this policy' and describes it as 'essential for the health and wellbeing of our students and of our community,' indicating clear approval of the regulation.", "tone": "positive", "tags": ["student wellbeing", "community support", "education policy"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87919", "analyzed_at": "2026-05-05T20:44:23.589115+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "Support from a School Counselor", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter explicitly states support for the guidance, noting it will be 'incredibly helpful' and 'important in order to better support transgender students.' This indicates a clear approval of the proposed regulation.", "tone": "positive", "tags": ["LGBTQ+ inclusion", "student support", "mental health", "school counseling"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87920", "analyzed_at": "2026-05-05T20:44:25.159983+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "I support this policy", "stance": "support", "stance_confidence": 0.95, "stance_rationale": "The commenter explicitly states 'I support this policy' and expresses belief in the importance of a 'welcoming and nurturing environment' for transgender students, indicating clear support for the regulation.", "tone": "positive", "tags": ["LGBTQ+ inclusion", "student safety"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87921", "analyzed_at": "2026-05-05T20:44:28.076212+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "It’s about time!", "stance": "support", "stance_confidence": 0.95, "stance_rationale": "The commenter expresses clear support for the regulation by stating that the guidance is \"a long time coming and is desperately needed.\" This indicates a strong desire for the regulation to be adopted to address issues faced by transgender students, like their son.", "tone": "positive", "tags": ["bullying prevention", "LGBTQ+ inclusion", "student safety"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87922", "analyzed_at": "2026-05-05T20:44:29.673172+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "A long overdue policy", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter expresses strong support for the policy, describing it as 'pro-equality' and 'evidence based,' and states that it would 'guarantee protections for transgender and gender variant youth.' The use of phrases like 'incredibly excited' and 'kudos to you, champions of equality!' further indicates a supportive stance.", "tone": "positive", "tags": ["LGBTQ+ inclusion", "student safety", "bullying prevention", "equality"], "truncated": false, "error": null}
+{"run_id": "e84adaf5-5250-42b9-97c1-59623bd99bc7", "forum_id": "452", "comment_id": "87923", "analyzed_at": "2026-05-05T20:44:35.056904+00:00", "model": "gpt-4o", "prompt_version": "cb41250", "input_title": "100% support", "stance": "support", "stance_confidence": 1.0, "stance_rationale": "The commenter explicitly states 'I totally support this needed policy,' indicating clear support for the regulation. They emphasize the importance of safety, support, and equality for all kids, aligning with the goals of the proposed regulation.", "tone": "positive", "tags": ["student safety", "LGBTQ+ inclusion", "nondiscrimination"], "truncated": false, "error": null}

analysis/gpt4o/runs/5b8714a7-0666-40a2-9d69-2d9ce9074406.json

@@ -0,0 +1,16 @@
+{
+  "run_id": "5b8714a7-0666-40a2-9d69-2d9ce9074406",
+  "input_filename": "output\\f452.jsonl",
+  "input_sha256": "59dcc8b13cc2a386977a8b934c498c7e639b7e684a94ca1bfd10a14878670018",
+  "prompt_hash": "cb41250",
+  "model": "gpt-4o",
+  "batch_id": "batch_69fa579c7cd081909c049715838df6c6",
+  "records_submitted": 9083,
+  "records_completed": 0,
+  "records_failed": 0,
+  "request_filename": "C:\\Users\\moses\\projects\\vath\\analysis\\gpt4o\\requests\\5b8714a7-0666-40a2-9d69-2d9ce9074406.jsonl",
+  "raw_output_filename": null,
+  "normalized_output_filename": null,
+  "created_at": "2026-05-05T20:48:28.268022+00:00",
+  "completed_at": null
+}

analysis/prompt-1.txt

@@ -0,0 +1,23 @@
+You are an expert policy analyst classifying public comments submitted to the Virginia Town Hall
+regulatory comment system. You will be given the text of a proposed regulation and a single
+public comment. Return ONLY a JSON object — no other text.
+
+Definitions:
+- stance: the commenter's position on whether the regulation should be adopted.
+  "support" = wants it approved (as-is or with changes);
+  "oppose"  = wants it rejected or substantially weakened;
+  "neutral" = takes no position, asks a question, or provides factual input only;
+  "unknown" = too vague, off-topic, or uninterpretable to classify.
+- tone: the emotional register of the writing, independent of stance.
+  "positive" = affirming, hopeful, appreciative;
+  "negative" = angry, fearful, alarmed, or contemptuous;
+  "neutral"  = matter-of-fact, procedural, or informational;
+  "mixed"    = contains both positive and negative emotional content;
+  "unclear"  = tone cannot be determined (e.g., a one-word comment).
+- stance_confidence: float 0.0-1.0, your confidence in the stance label.
+- stance_rationale: 1-3 sentences explaining the key evidence; quote specific phrases where possible.
+- tags: up to 5 short topic labels relevant to the comment's specific concerns (e.g.
+  "parental rights", "student safety", "privacy", "religious freedom", "LGBTQ+ inclusion",
+  "bullying prevention", "school sports", "bathroom access"). Empty array if none apply.
+
+Return exactly these keys: stance, stance_confidence, stance_rationale, tone, tags.

docs/openai-batch.md

@@ -0,0 +1,384 @@
+# Batch API
+
+Learn how to use OpenAI's Batch API to send asynchronous groups of requests with 50% lower costs, a separate pool of significantly higher rate limits, and a clear 24-hour turnaround time. The service is ideal for processing jobs that don't require immediate responses. You can also [explore the API reference directly here](https://developers.openai.com/api/docs/api-reference/batch).
+
+## Overview
+
+While some uses of the OpenAI Platform require you to send synchronous requests, there are many cases where requests do not need an immediate response or [rate limits](https://developers.openai.com/api/docs/guides/rate-limits) prevent you from executing a large number of queries quickly. Batch processing jobs are often helpful in use cases like:
+
+1. Running evaluations
+2. Classifying large datasets
+3. Embedding content repositories
+4. Queuing large offline video-render jobs
+
+The Batch API offers a straightforward set of endpoints that allow you to collect a set of requests into a single file, kick off a batch processing job to execute these requests, query for the status of that batch while the underlying requests execute, and eventually retrieve the collected results when the batch is complete.
+
+Compared to using standard endpoints directly, Batch API has:
+
+1. **Better cost efficiency:** 50% cost discount compared to synchronous APIs
+2. **Higher rate limits:** [Substantially more headroom](https://platform.openai.com/settings/organization/limits) compared to the synchronous APIs
+3. **Fast completion times:** Each batch completes within 24 hours (and often more quickly)
+
+## Getting started
+
+### 1. Prepare your batch file
+
+Batches start with a `.jsonl` file where each line contains the details of an individual request to the API. For now, the available endpoints are:
+
+- `/v1/responses` ([Responses API](https://developers.openai.com/api/docs/api-reference/responses))
+- `/v1/chat/completions` ([Chat Completions API](https://developers.openai.com/api/docs/api-reference/chat))
+- `/v1/embeddings` ([Embeddings API](https://developers.openai.com/api/docs/api-reference/embeddings))
+- `/v1/completions` ([Completions API](https://developers.openai.com/api/docs/api-reference/completions))
+- `/v1/moderations` ([Moderations guide](https://developers.openai.com/api/docs/guides/moderation))
+- `/v1/images/generations` ([Images API](https://developers.openai.com/api/docs/api-reference/images))
+- `/v1/images/edits` ([Images API](https://developers.openai.com/api/docs/api-reference/images))
+- `/v1/videos` ([Video generation guide](https://developers.openai.com/api/docs/guides/video-generation))
+
+For a given input file, the parameters in each line's `body` field are the same as the parameters for the underlying endpoint. Each request must include a unique `custom_id` value, which you can use to reference results after completion. Here's an example of an input file with 2 requests. Note that each input file can only include requests to a single model.
+
+For video generation in Batch:
+
+- Batch currently supports `POST /v1/videos` only.
+- Batch requests for videos must use JSON, not multipart.
+- Upload assets ahead of time and pass supported asset references in the request body rather than using multipart uploads.
+- Use `input_reference` for image-guided generations in Batch. In JSON requests, pass `input_reference` as an object with either `file_id` or `image_url`.
+- Multipart `input_reference` uploads, including video reference inputs, aren't supported in Batch.
+- Batch-generated videos are available for download for up to `24` hours after the batch completes.
+
+When targeting `/v1/moderations`, include an `input` field in every request body. Batch accepts both plain-text inputs (for `omni-moderation-latest` and `text-moderation-latest`) and multimodal content arrays (for `omni-moderation-latest`). The Batch worker enforces the same non-streaming requirement as the synchronous Moderations API and rejects requests that set `stream=true`.
+
+```jsonl
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
+```
+
+#### Moderations input examples
+
+Text-only request:
+
+```jsonl
+{
+  "custom_id": "moderation-text-1",
+  "method": "POST",
+  "url": "/v1/moderations",
+  "body": {
+    "model": "omni-moderation-latest",
+    "input": "This is a harmless test sentence."
+  }
+}
+```
+
+Multimodal request:
+
+```jsonl
+{
+  "custom_id": "moderation-mm-1",
+  "method": "POST",
+  "url": "/v1/moderations",
+  "body": {
+    "model": "omni-moderation-latest",
+    "input": [
+      {
+        "type": "text",
+        "text": "Describe this image"
+      },
+      {
+        "type": "image_url",
+        "image_url": {
+          "url": "https://api.nga.gov/iiif/a2e6da57-3cd1-4235-b20e-95dcaefed6c8/full/!800,800/0/default.jpg"
+        }
+      }
+    ]
+  }
+}
+```
+
+Prefer referencing remote assets with `image_url` (instead of base64 blobs) to
+  keep your `.jsonl` files well below the 200 MB Batch upload limit,
+  especially for multimodal Moderations requests.
+
+### 2. Upload your batch input file
+
+Similar to our [Fine-tuning API](https://developers.openai.com/api/docs/guides/model-optimization), you must first upload your input file so that you can reference it correctly when kicking off batches. Upload your `.jsonl` file using the [Files API](https://developers.openai.com/api/docs/api-reference/files).
+
+Upload files for Batch API
+
+```javascript
+import fs from "fs";
+import OpenAI from "openai";
+const openai = new OpenAI();
+
+const file = await openai.files.create({
+  file: fs.createReadStream("batchinput.jsonl"),
+  purpose: "batch",
+});
+
+console.log(file);
+```
+
+```python
+from openai import OpenAI
+client = OpenAI()
+
+batch_input_file = client.files.create(
+    file=open("batchinput.jsonl", "rb"),
+    purpose="batch"
+)
+
+print(batch_input_file)
+```
+
+```bash
+curl https://api.openai.com/v1/files \\
+  -H "Authorization: Bearer $OPENAI_API_KEY" \\
+  -F purpose="batch" \\
+  -F file="@batchinput.jsonl"
+```
+
+
+### 3. Create the batch
+
+Once you've successfully uploaded your input file, you can use the input File object's ID to create a batch. In this case, let's assume the file ID is `file-abc123`. For now, the completion window can only be set to `24h`. You can also provide custom metadata via an optional `metadata` parameter.
+
+Create the Batch
+
+```javascript
+import OpenAI from "openai";
+const openai = new OpenAI();
+
+const batch = await openai.batches.create({
+  input_file_id: "file-abc123",
+  endpoint: "/v1/chat/completions",
+  completion_window: "24h"
+});
+
+console.log(batch);
+```
+
+```python
+from openai import OpenAI
+client = OpenAI()
+
+batch_input_file_id = batch_input_file.id
+client.batches.create(
+    input_file_id=batch_input_file_id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={
+        "description": "nightly eval job"
+    }
+)
+```
+
+```bash
+curl https://api.openai.com/v1/batches \\
+  -H "Authorization: Bearer $OPENAI_API_KEY" \\
+  -H "Content-Type: application/json" \\
+  -d '{
+    "input_file_id": "file-abc123",
+    "endpoint": "/v1/chat/completions",
+    "completion_window": "24h"
+  }'
+```
+
+
+This request will return a [Batch object](https://developers.openai.com/api/docs/api-reference/batch/object) with metadata about your batch:
+
+```python
+{
+  "id": "batch_abc123",
+  "object": "batch",
+  "endpoint": "/v1/chat/completions",
+  "errors": null,
+  "input_file_id": "file-abc123",
+  "completion_window": "24h",
+  "status": "validating",
+  "output_file_id": null,
+  "error_file_id": null,
+  "created_at": 1714508499,
+  "in_progress_at": null,
+  "expires_at": 1714536634,
+  "completed_at": null,
+  "failed_at": null,
+  "expired_at": null,
+  "request_counts": {
+    "total": 0,
+    "completed": 0,
+    "failed": 0
+  },
+  "metadata": null
+}
+```
+
+### 4. Check the status of a batch
+
+You can check the status of a batch at any time, which will also return a Batch object.
+
+Check the status of a batch
+
+```javascript
+import OpenAI from "openai";
+const openai = new OpenAI();
+
+const batch = await openai.batches.retrieve("batch_abc123");
+console.log(batch);
+```
+
+```python
+from openai import OpenAI
+client = OpenAI()
+
+batch = client.batches.retrieve("batch_abc123")
+print(batch)
+```
+
+```bash
+curl https://api.openai.com/v1/batches/batch_abc123 \\
+  -H "Authorization: Bearer $OPENAI_API_KEY" \\
+  -H "Content-Type: application/json"
+```
+
+
+The status of a given Batch object can be any of the following:
+
+| Status        | Description                                                                    |
+| ------------- | ------------------------------------------------------------------------------ |
+| `validating`  | the input file is being validated before the batch can begin                   |
+| `failed`      | the input file has failed the validation process                               |
+| `in_progress` | the input file was successfully validated and the batch is currently being run |
+| `finalizing`  | the batch has completed and the results are being prepared                     |
+| `completed`   | the batch has been completed and the results are ready                         |
+| `expired`     | the batch was not able to be completed within the 24-hour time window          |
+| `cancelling`  | the batch is being cancelled (may take up to 10 minutes)                       |
+| `cancelled`   | the batch was cancelled                                                        |
+
+### 5. Retrieve the results
+
+Once the batch is complete, you can download the output by making a request against the [Files API](https://developers.openai.com/api/docs/api-reference/files) via the `output_file_id` field from the Batch object and writing it to a file on your machine, in this case `batch_output.jsonl`
+
+Retrieving the batch results
+
+```javascript
+import OpenAI from "openai";
+const openai = new OpenAI();
+
+const fileResponse = await openai.files.content("file-xyz123");
+const fileContents = await fileResponse.text();
+
+console.log(fileContents);
+```
+
+```python
+from openai import OpenAI
+client = OpenAI()
+
+file_response = client.files.content("file-xyz123")
+print(file_response.text)
+```
+
+```bash
+curl https://api.openai.com/v1/files/file-xyz123/content \\
+  -H "Authorization: Bearer $OPENAI_API_KEY" > batch_output.jsonl
+```
+
+
+The output `.jsonl` file will have one response line for every successful request line in the input file. Any failed requests in the batch will have their error information written to an error file that can be found via the batch's `error_file_id`.
+
+For `/v1/videos`, a completed batch result contains video objects that have already reached a terminal state such as `completed`, `failed`, or `expired`. You can use the returned video IDs to download final assets immediately after the batch finishes.
+
+Note that the output line order **may not match** the input line order.
+  Instead of relying on order to process your results, use the custom_id field
+  which will be present in each line of your output file and allow you to map
+  requests in your input to results in your output.
+
+```jsonl
+{"id": "batch_req_123", "custom_id": "request-2", "response": {"status_code": 200, "request_id": "req_123", "body": {"id": "chatcmpl-123", "object": "chat.completion", "created": 1711652795, "model": "gpt-3.5-turbo-0125", "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 22, "completion_tokens": 2, "total_tokens": 24}, "system_fingerprint": "fp_123"}}, "error": null}
+{"id": "batch_req_456", "custom_id": "request-1", "response": {"status_code": 200, "request_id": "req_789", "body": {"id": "chatcmpl-abc", "object": "chat.completion", "created": 1711652789, "model": "gpt-3.5-turbo-0125", "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello! How can I assist you today?"}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 20, "completion_tokens": 9, "total_tokens": 29}, "system_fingerprint": "fp_3ba"}}, "error": null}
+```
+
+The output file will automatically be deleted 30 days after the batch is complete.
+
+### 6. Cancel a batch
+
+If necessary, you can cancel an ongoing batch. The batch's status will change to `cancelling` until in-flight requests are complete (up to 10 minutes), after which the status will change to `cancelled`.
+
+Cancelling a batch
+
+```javascript
+import OpenAI from "openai";
+const openai = new OpenAI();
+
+const batch = await openai.batches.cancel("batch_abc123");
+console.log(batch);
+```
+
+```python
+from openai import OpenAI
+client = OpenAI()
+
+client.batches.cancel("batch_abc123")
+```
+
+```bash
+curl https://api.openai.com/v1/batches/batch_abc123/cancel \\
+  -H "Authorization: Bearer $OPENAI_API_KEY" \\
+  -H "Content-Type: application/json" \\
+  -X POST
+```
+
+
+### 7. Get a list of all batches
+
+At any time, you can see all your batches. For users with many batches, you can use the `limit` and `after` parameters to paginate your results.
+
+Getting a list of all batches
+
+```javascript
+import OpenAI from "openai";
+const openai = new OpenAI();
+
+const list = await openai.batches.list();
+
+for await (const batch of list) {
+  console.log(batch);
+}
+```
+
+```python
+from openai import OpenAI
+client = OpenAI()
+
+client.batches.list(limit=10)
+```
+
+```bash
+curl https://api.openai.com/v1/batches?limit=10 \\
+  -H "Authorization: Bearer $OPENAI_API_KEY" \\
+  -H "Content-Type: application/json"
+```
+
+
+## Model availability
+
+The Batch API is widely available across most of our models, but not all. Please refer to the [model reference docs](https://developers.openai.com/api/docs/models) to ensure the model you're using supports the Batch API.
+
+## Rate limits
+
+Batch API rate limits are separate from existing per-model rate limits. The Batch API has three types of rate limits:
+
+1. **Per-batch limits:** A single batch may include up to 50,000 requests, and a batch input file can be up to 200 MB in size. Note that `/v1/embeddings` batches are also restricted to a maximum of 50,000 embedding inputs across all requests in the batch.
+2. **Enqueued prompt tokens per model:** Each model has a maximum number of enqueued prompt tokens allowed for batch processing. You can find these limits on the [Platform Settings page](https://platform.openai.com/settings/organization/limits).
+3. **Batch creation rate limit:** You can create up to 2,000 batches per hour. If you need to submit more requests, increase the number of requests per batch.
+
+There are no limits for output tokens for the Batch API today. Because Batch API rate limits are a new, separate pool, **using the Batch API will not consume tokens from your standard per-model rate limits**, thereby offering you a convenient way to increase the number of requests and processed tokens you can use when querying our API.
+
+## Batch expiration
+
+Batches that do not complete in time eventually move to an `expired` state; unfinished requests within that batch are cancelled, and any responses to completed requests are made available via the batch's output file. You will be charged for tokens consumed from any completed requests.
+
+Expired requests will be written to your error file with the message as shown below. You can use the `custom_id` to retrieve the request data for expired requests.
+
+```jsonl
+{"id": "batch_req_123", "custom_id": "request-3", "response": null, "error": {"code": "batch_expired", "message": "This request could not be executed before the completion window expired."}}
+{"id": "batch_req_123", "custom_id": "request-7", "response": null, "error": {"code": "batch_expired", "message": "This request could not be executed before the completion window expired."}}
+```

docs/tasks.org

@@ -1,3 +1,7 @@
+#+title: VATH Task Log
+#+date: [2026-05-05 Tue]
+#+startup: Overview
+
 * [X] t1.1: scrape one forum (1)
 Use https://www.townhall.virginia.gov/L/comments.cfm?GDocForumID=452 as the first forum. Scraper should be run manually at this step.
 ViewComments (townhall.virginia.gov/L/ViewComments.cfm?CommentID=#) appears to be raw list of all comments on forum - could be useful later for whole-scrape
@@ -33,7 +37,7 @@ Comments are hydrated in backend via js-cued button (AJAX?).
    - retrieved 9083 comments
 - datetime: [2026-05-05 Tue 14:00]
 
-* [ ] t1.2: initial 4o sentiment
+* [X] t1.2: initial 4o sentiment
 Write a simple manual pipeline for gpt-4o that reads one scraped forum jsonl file and roduces a separate analyzed jsonl file. this step must not mutate scraper output. analysis should classify each comment for regulatory stance, generic tone/sentiment, confidence, and enough rationale/evidence to support later dashboard drilldown.
 Should be run manually, separate from scraper. You may use scrapy, but are not required to.
 - Sentiment is derived, not scraped - keep separate from raw comments.
@@ -68,20 +72,38 @@ Should be run manually, separate from scraper. You may use scrapy, but are not r
 
 ** evidence
 - commit: d834d18
-- tests: 20 passing (pytest tests/test_gpt4o_analysis.py), 28 total across suite
-  python ./analysis/gpt4o/analysis.py --limit 5 ./output/f452.jsonl
+- tests: 20 passing (pytest tests/analysis_gpt4o_realtime.py), 28 total across suite
+  - `python ./analysis/gpt4o/analysis_realtime.py --limit 5 ./output/f452.jsonl`
+  - see: ./analysis/gpt4o/forum452_unknown_gpt-4o_2026-05-05T18-48-32+00-00.jsonl
 - date: [2026-05-05 Tue 15:00]
 
-* [ ] t1.2.1: 4o with batch processing
+* [ ] t1.2.1: batch processing
+Create analysis-batch.py to capture same elements as t1.2 above. 
+May need to add multiple commands to upload, check batch status, download, etc.
+Commands should all be run manually.
+Reference: ./docs/openai-batch.md. openai batch output order is not guaranteed, so custom_id is mandatory for reconciliation
 ** acceptance criteria
 1. input scraped jsonl doc by filename/path, and process the whole thing via batch processing
-
+   - ignore non-comment items in jsonl
+   - do not modify raw scraper output
+   - specify model and prompt
+2. output a run manifest in ./analysis/<model>/runs/<run_id>.json
+   - include: include run_id, input_filename, input_sha256, prompt_hash, model, batch_id, records_submitted, records_completed, records_failed, request_filename, raw_output_filename, normalized_output_filename, created_at, completed_at
+3. add tests without live api calls
 ** notes
+- analysis/gpt4o/analysis-batch.py with three subcommands:
+  - `submit`: reads scraped JSONL, builds batch request file (requests/<run_id>.jsonl), uploads to Files API, creates batch, saves manifest to runs/<run_id>.json. Prints run_id to stdout for scripting.
+  - `status`: retrieves batch from OpenAI, prints status + counts, updates manifest.
+  - `download`: downloads raw output to raw/<run_id>.jsonl, normalizes to <run_id>_<model>.jsonl using comment_lookup keyed by comment_id for reconciliation (batch output order not guaranteed). Updates manifest with filenames, counts, completed_at.
+- custom_id format: comment_{comment_id} — unique within a forum, stable across runs.
+- PROMPT_VERSION derived from analysis/prompt-1.txt (same file as realtime); both scripts produce matching prompt_hash in all records.
+- analysis/prompt-1.txt: system prompt as plaintext, read at import time by both scripts. Edit here to change prompt for both pipelines.
+- Tests use importlib.util to load hyphenated filenames; monkeypatch for RUNS_DIR in save/load test.
 
 ** evidence
 - commit:
-- tests:
-- date:
+- tests: 18 passing (pytest tests/analysis_gpt4o_batch.py), 46 total across suite
+- datetime: [2026-05-05 Tue 17:00]
 
 * [ ] X: complete proposal information
 Ensure we capture as much useful information as possible about the actual proposal - contact information, etc. what the state actually says about what was posted. 

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+testpaths = tests
+python_files = *.py
+python_classes = Test*
+python_functions = test_*

tests/analysis_gpt4o_batch.py

@@ -0,0 +1,252 @@
+"""Unit tests for analysis/gpt4o/analysis_batch.py — no real API calls."""
+
+import json
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "analysis" / "gpt4o"))
+import analysis_batch as bt
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+
+FORUM_ITEM = {
+    "forum_id": "452",
+    "reg_title": "Model Policies for Transgender Students",
+    "reg_desc": "Guidance developed in response to HB 145.",
+}
+
+COMMENT_ITEM = {
+    "forum_id": "452",
+    "comment_id": "87914",
+    "author": "Alice Example",
+    "date": "2021-01-04T09:15:00",
+    "title": "I support this policy",
+    "text": "This is a great policy that protects students.",
+}
+
+RAW_SUCCESS_LINE = {
+    "id": "batch_req_001",
+    "custom_id": "comment_87914",
+    "response": {
+        "status_code": 200,
+        "request_id": "req_abc",
+        "body": {
+            "id": "chatcmpl-xyz",
+            "choices": [{
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": json.dumps({
+                        "stance": "support",
+                        "stance_confidence": 0.95,
+                        "stance_rationale": "Commenter explicitly endorses the policy.",
+                        "tone": "positive",
+                        "tags": ["student safety"],
+                    }),
+                },
+                "finish_reason": "stop",
+            }],
+        },
+    },
+    "error": None,
+}
+
+RAW_ERROR_LINE = {
+    "id": "batch_req_002",
+    "custom_id": "comment_87914",
+    "response": None,
+    "error": {"code": "batch_expired", "message": "This request could not be executed."},
+}
+
+RAW_HTTP_ERROR_LINE = {
+    "id": "batch_req_003",
+    "custom_id": "comment_87914",
+    "response": {"status_code": 400, "body": {}},
+    "error": None,
+}
+
+COMMENT_LOOKUP = {"87914": COMMENT_ITEM}
+ANALYZED_AT = "2026-05-05T18:00:00+00:00"
+RUN_ID = "test-run-id-123"
+MODEL = "gpt-4o"
+
+
+# ---------------------------------------------------------------------------
+# Prompt versioning (batch reads the same prompt file)
+
+def test_prompt_version_is_7_hex_chars():
+    assert len(bt.PROMPT_VERSION) == 7
+    assert all(c in "0123456789abcdef" for c in bt.PROMPT_VERSION)
+
+
+def test_prompt_version_matches_realtime():
+    """Both scripts must derive the same PROMPT_VERSION from the same file."""
+    import analysis_realtime as rt
+    assert bt.PROMPT_VERSION == rt.PROMPT_VERSION
+
+
+# ---------------------------------------------------------------------------
+# custom_id helpers
+
+def test_custom_id_from():
+    assert bt.custom_id_from("87914") == "comment_87914"
+
+
+def test_parse_custom_id():
+    assert bt.parse_custom_id("comment_87914") == "87914"
+
+
+def test_custom_id_round_trip():
+    cid = "12345"
+    assert bt.parse_custom_id(bt.custom_id_from(cid)) == cid
+
+
+# ---------------------------------------------------------------------------
+# build_batch_request_line
+
+def test_batch_request_line_structure():
+    line = bt.build_batch_request_line(COMMENT_ITEM, FORUM_ITEM, "gpt-4o")
+    assert line["custom_id"] == "comment_87914"
+    assert line["method"] == "POST"
+    assert line["url"] == "/v1/chat/completions"
+    assert line["body"]["model"] == "gpt-4o"
+    assert line["body"]["temperature"] == 0.0
+    assert line["body"]["response_format"] == {"type": "json_object"}
+    messages = line["body"]["messages"]
+    assert messages[0]["role"] == "system"
+    assert messages[1]["role"] == "user"
+
+
+def test_batch_request_line_includes_reg_context():
+    line = bt.build_batch_request_line(COMMENT_ITEM, FORUM_ITEM, "gpt-4o")
+    user_content = line["body"]["messages"][1]["content"]
+    assert "Model Policies for Transgender Students" in user_content
+    assert "HB 145" in user_content
+
+
+def test_batch_request_line_truncation():
+    long_comment = {**COMMENT_ITEM, "text": "x" * 7000}
+    line = bt.build_batch_request_line(long_comment, FORUM_ITEM, "gpt-4o")
+    user_content = line["body"]["messages"][1]["content"]
+    assert "... [truncated]" in user_content
+    assert user_content.count("x") == bt.MAX_COMMENT_CHARS
+
+
+# ---------------------------------------------------------------------------
+# normalize_output_line — success
+
+def test_normalize_success_all_keys():
+    record = bt.normalize_output_line(RAW_SUCCESS_LINE, COMMENT_LOOKUP, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    required = {
+        "run_id", "forum_id", "comment_id", "analyzed_at", "model", "prompt_version",
+        "stance", "stance_confidence", "stance_rationale", "tone", "tags",
+        "input_title", "truncated", "error",
+    }
+    assert required == set(record.keys())
+
+
+def test_normalize_success_values():
+    record = bt.normalize_output_line(RAW_SUCCESS_LINE, COMMENT_LOOKUP, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    assert record["stance"] == "support"
+    assert record["tone"] == "positive"
+    assert record["comment_id"] == "87914"
+    assert record["run_id"] == RUN_ID
+    assert record["analyzed_at"] == ANALYZED_AT
+    assert record["error"] is None
+    assert record["truncated"] is False
+
+
+def test_normalize_success_input_title():
+    record = bt.normalize_output_line(RAW_SUCCESS_LINE, COMMENT_LOOKUP, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    assert record["input_title"] == COMMENT_ITEM["title"]
+
+
+# ---------------------------------------------------------------------------
+# normalize_output_line — errors
+
+def test_normalize_batch_expired_error():
+    record = bt.normalize_output_line(RAW_ERROR_LINE, COMMENT_LOOKUP, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    assert record["error"] is not None
+    assert "could not be executed" in record["error"]
+    assert record["stance"] is None
+    assert record["tone"] is None
+
+
+def test_normalize_http_error():
+    record = bt.normalize_output_line(RAW_HTTP_ERROR_LINE, COMMENT_LOOKUP, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    assert record["error"] is not None
+    assert record["stance"] is None
+
+
+def test_normalize_malformed_json_in_response():
+    bad_line = {
+        "id": "batch_req_004",
+        "custom_id": "comment_87914",
+        "response": {
+            "status_code": 200,
+            "body": {"choices": [{"message": {"content": "not valid json{{{"}}]},
+        },
+        "error": None,
+    }
+    record = bt.normalize_output_line(bad_line, COMMENT_LOOKUP, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    assert record["error"] is not None
+    assert record["stance"] is None
+
+
+def test_normalize_unknown_comment_id():
+    """A custom_id not in lookup yields empty forum_id and title but doesn't crash."""
+    record = bt.normalize_output_line(RAW_SUCCESS_LINE, {}, RUN_ID, ANALYZED_AT, MODEL, bt.PROMPT_VERSION)
+    assert record["comment_id"] == "87914"
+    assert record["forum_id"] == ""
+    assert record["input_title"] == ""
+
+
+# ---------------------------------------------------------------------------
+# Manifest
+
+def test_make_manifest_all_keys():
+    m = bt.make_manifest(
+        run_id=RUN_ID,
+        input_filename="output/forum452.jsonl",
+        input_sha256="abc123",
+        model="gpt-4o",
+        batch_id="batch_xyz",
+        records_submitted=100,
+        request_filename="analysis/gpt4o/requests/test-run-id-123.jsonl",
+    )
+    required = {
+        "run_id", "input_filename", "input_sha256", "prompt_hash", "model",
+        "batch_id", "records_submitted", "records_completed", "records_failed",
+        "request_filename", "raw_output_filename", "normalized_output_filename",
+        "created_at", "completed_at",
+    }
+    assert required == set(m.keys())
+
+
+def test_make_manifest_initial_nulls():
+    m = bt.make_manifest(
+        run_id=RUN_ID, input_filename="f", input_sha256="s",
+        model="gpt-4o", batch_id="b", records_submitted=10, request_filename="r",
+    )
+    assert m["records_completed"] is None
+    assert m["records_failed"] is None
+    assert m["raw_output_filename"] is None
+    assert m["normalized_output_filename"] is None
+    assert m["completed_at"] is None
+    assert m["prompt_hash"] == bt.PROMPT_VERSION
+
+
+def test_manifest_save_load_roundtrip(tmp_path, monkeypatch):
+    monkeypatch.setattr(bt, "RUNS_DIR", tmp_path)
+    m = bt.make_manifest(
+        run_id=RUN_ID, input_filename="f", input_sha256="s",
+        model="gpt-4o", batch_id="b", records_submitted=42, request_filename="r",
+    )
+    bt.save_manifest(m)
+    loaded = bt.load_manifest(RUN_ID)
+    assert loaded == m

tests/analysis_gpt4o_realtime.py

@@ -1,15 +1,14 @@
-"""Unit tests for analysis/gpt4o/analysis.py — no real API calls."""
+"""Unit tests for analysis/gpt4o/analysis_realtime.py — no real API calls."""
 
 import json
 import sys
 from pathlib import Path
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock
 
 import pytest
 
-# Make the module importable without installing as a package
 sys.path.insert(0, str(Path(__file__).parent.parent / "analysis" / "gpt4o"))
-import analysis as gpt4o
+import analysis_realtime as rt
 
 
 # ---------------------------------------------------------------------------
@@ -51,26 +50,25 @@ def _mock_client(response_content: str = MOCK_RESPONSE_CONTENT):
 # Prompt versioning
 
 def test_prompt_version_is_7_hex_chars():
-    assert len(gpt4o.PROMPT_VERSION) == 7
-    assert all(c in "0123456789abcdef" for c in gpt4o.PROMPT_VERSION)
+    assert len(rt.PROMPT_VERSION) == 7
+    assert all(c in "0123456789abcdef" for c in rt.PROMPT_VERSION)
 
 
-def test_prompt_version_changes_with_system_prompt():
+def test_prompt_version_matches_prompt_file():
     import hashlib
-    alt = hashlib.sha256(("CHANGED" + gpt4o.USER_TEMPLATE).encode("utf-8")).hexdigest()[:7]
-    assert alt != gpt4o.PROMPT_VERSION
+    prompt_file = Path(__file__).parent.parent / "analysis" / "prompt-1.txt"
+    expected = hashlib.sha256(prompt_file.read_text(encoding="utf-8").strip().encode()).hexdigest()[:7]
+    assert rt.PROMPT_VERSION == expected
 
 
 def test_prompt_version_is_stable():
     import hashlib
-    v2 = hashlib.sha256(
-        (gpt4o.SYSTEM_PROMPT + gpt4o.USER_TEMPLATE).encode("utf-8")
-    ).hexdigest()[:7]
-    assert v2 == gpt4o.PROMPT_VERSION
+    v2 = hashlib.sha256(rt.SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:7]
+    assert v2 == rt.PROMPT_VERSION
 
 
 # ---------------------------------------------------------------------------
-# Item detection via load_items
+# load_items
 
 def test_load_items_separates_forum_and_comments(tmp_path):
     jsonl = tmp_path / "test.jsonl"
@@ -78,7 +76,7 @@ def test_load_items_separates_forum_and_comments(tmp_path):
         json.dumps(FORUM_ITEM) + "\n" + json.dumps(COMMENT_ITEM) + "\n",
         encoding="utf-8",
     )
-    forum, comments = gpt4o.load_items(jsonl)
+    forum, comments = rt.load_items(jsonl)
     assert forum is not None
     assert forum["reg_title"] == FORUM_ITEM["reg_title"]
     assert len(comments) == 1
@@ -88,18 +86,15 @@ def test_load_items_separates_forum_and_comments(tmp_path):
 def test_load_items_no_forum(tmp_path):
     jsonl = tmp_path / "test.jsonl"
     jsonl.write_text(json.dumps(COMMENT_ITEM) + "\n", encoding="utf-8")
-    forum, comments = gpt4o.load_items(jsonl)
+    forum, comments = rt.load_items(jsonl)
     assert forum is None
     assert len(comments) == 1
 
 
 def test_load_items_skips_blank_lines(tmp_path):
     jsonl = tmp_path / "test.jsonl"
-    jsonl.write_text(
-        "\n" + json.dumps(COMMENT_ITEM) + "\n\n",
-        encoding="utf-8",
-    )
-    _, comments = gpt4o.load_items(jsonl)
+    jsonl.write_text("\n" + json.dumps(COMMENT_ITEM) + "\n\n", encoding="utf-8")
+    _, comments = rt.load_items(jsonl)
     assert len(comments) == 1
 
 
@@ -108,40 +103,37 @@ def test_load_items_skips_blank_lines(tmp_path):
 
 def test_truncation_applied():
     long_comment = {**COMMENT_ITEM, "text": "x" * 7000}
-    messages, truncated = gpt4o.build_messages(long_comment, FORUM_ITEM)
+    messages, truncated = rt.build_messages(long_comment, FORUM_ITEM)
     assert truncated is True
-    user_content = messages[1]["content"]
-    assert "... [truncated]" in user_content
-    # The x's in the prompt must not exceed MAX_COMMENT_CHARS
-    x_count = user_content.count("x")
-    assert x_count == gpt4o.MAX_COMMENT_CHARS
+    assert "... [truncated]" in messages[1]["content"]
+    assert messages[1]["content"].count("x") == rt.MAX_COMMENT_CHARS
 
 
 def test_no_truncation_for_short_comment():
-    _, truncated = gpt4o.build_messages(COMMENT_ITEM, FORUM_ITEM)
+    _, truncated = rt.build_messages(COMMENT_ITEM, FORUM_ITEM)
     assert truncated is False
 
 
 def test_empty_text_fallback():
     empty = {**COMMENT_ITEM, "text": ""}
-    messages, truncated = gpt4o.build_messages(empty, FORUM_ITEM)
+    messages, truncated = rt.build_messages(empty, FORUM_ITEM)
     assert "[No body text provided]" in messages[1]["content"]
     assert truncated is False
 
 
 def test_none_text_fallback():
     none_text = {**COMMENT_ITEM, "text": None}
-    messages, _ = gpt4o.build_messages(none_text, FORUM_ITEM)
+    messages, _ = rt.build_messages(none_text, FORUM_ITEM)
     assert "[No body text provided]" in messages[1]["content"]
 
 
 def test_missing_forum_uses_unknown_context():
-    messages, _ = gpt4o.build_messages(COMMENT_ITEM, None)
+    messages, _ = rt.build_messages(COMMENT_ITEM, None)
     assert "[unknown]" in messages[1]["content"]
 
 
 def test_reg_context_included_in_prompt():
-    messages, _ = gpt4o.build_messages(COMMENT_ITEM, FORUM_ITEM)
+    messages, _ = rt.build_messages(COMMENT_ITEM, FORUM_ITEM)
     assert FORUM_ITEM["reg_title"] in messages[1]["content"]
     assert "HB 145" in messages[1]["content"]
 
@@ -150,8 +142,7 @@ def test_reg_context_included_in_prompt():
 # Output record schema
 
 def test_output_record_all_keys_present():
-    client = _mock_client()
-    record = gpt4o.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
+    record = rt.analyze_comment(_mock_client(), COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
     required = {
         "run_id", "forum_id", "comment_id", "analyzed_at", "model", "prompt_version",
         "stance", "stance_confidence", "stance_rationale", "tone", "tags",
@@ -161,8 +152,7 @@ def test_output_record_all_keys_present():
 
 
 def test_output_record_correct_types():
-    client = _mock_client()
-    record = gpt4o.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
+    record = rt.analyze_comment(_mock_client(), COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
     assert record["stance"] == "support"
     assert isinstance(record["stance_confidence"], float)
     assert isinstance(record["tags"], list)
@@ -171,13 +161,12 @@ def test_output_record_correct_types():
 
 
 def test_output_record_metadata():
-    client = _mock_client()
-    record = gpt4o.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
+    record = rt.analyze_comment(_mock_client(), COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
     assert record["run_id"] == "run-123"
     assert record["forum_id"] == "452"
     assert record["comment_id"] == "87914"
     assert record["model"] == "gpt-4o"
-    assert record["prompt_version"] == gpt4o.PROMPT_VERSION
+    assert record["prompt_version"] == rt.PROMPT_VERSION
     assert record["input_title"] == COMMENT_ITEM["title"]
 
 
@@ -185,12 +174,12 @@ def test_output_record_metadata():
 # Error handling
 
 def test_error_record_on_api_failure():
-    client = MagicMock()
     import openai as _openai
+    client = MagicMock()
     client.chat.completions.create.side_effect = _openai.RateLimitError(
         "rate limit", response=MagicMock(status_code=429), body={}
     )
-    record = gpt4o.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
+    record = rt.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
     assert record["error"] is not None
     assert record["stance"] is None
     assert record["tone"] is None
@@ -198,8 +187,7 @@ def test_error_record_on_api_failure():
 
 
 def test_error_record_on_bad_json():
-    client = _mock_client("not valid json{{{")
-    record = gpt4o.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
+    record = rt.analyze_comment(_mock_client("not valid json{{{"), COMMENT_ITEM, FORUM_ITEM, "run-123", "gpt-4o")
     assert record["error"] is not None
     assert record["stance"] is None
 
@@ -210,21 +198,18 @@ def test_error_record_on_bad_json():
 def test_run_id_is_shared_across_records():
     client = _mock_client()
     run_id = "fixed-run-id"
-    r1 = gpt4o.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, run_id, "gpt-4o")
-    r2 = gpt4o.analyze_comment(client, {**COMMENT_ITEM, "comment_id": "99999"}, FORUM_ITEM, run_id, "gpt-4o")
+    r1 = rt.analyze_comment(client, COMMENT_ITEM, FORUM_ITEM, run_id, "gpt-4o")
+    r2 = rt.analyze_comment(client, {**COMMENT_ITEM, "comment_id": "99999"}, FORUM_ITEM, run_id, "gpt-4o")
     assert r1["run_id"] == r2["run_id"] == run_id
 
 
 # ---------------------------------------------------------------------------
-# Filename parsing
+# Filename helpers
 
 def test_scrape_ts_extracted_from_filename():
     p = Path("output/forum452_comments_2026-05-05T17-33-54+00-00.jsonl")
-    ts = gpt4o._scrape_ts_from_filename(p)
-    assert ts == "2026-05-05T17-33-54+00-00"
+    assert rt._scrape_ts_from_filename(p) == "2026-05-05T17-33-54+00-00"
 
 
 def test_scrape_ts_fallback_for_unknown_filename():
-    p = Path("output/somefile.jsonl")
-    ts = gpt4o._scrape_ts_from_filename(p)
-    assert ts == "unknown"
+    assert rt._scrape_ts_from_filename(Path("output/somefile.jsonl")) == "unknown"