updated scope to prep for costco scraper

README.md

@@ -0,0 +1,103 @@
+# scrape-giant
+
+Small grocery-history pipeline for Giant receipts.
+
+The project currently does four things:
+
+1. scrape Giant in-store order history from an active Firefox session
+2. enrich raw line items into a deterministic `items_enriched.csv`
+3. aggregate retailer-facing observed products and build a manual review queue
+4. create a first-pass canonical product layer plus conservative auto-links
+
+The work so far is Giant-specific on the ingest side and intentionally simple on
+the shared product-model side.
+
+## Current flow
+
+Run the commands from the repo root with the project venv active, or call them
+directly through `./venv/bin/python`.
+
+```bash
+./venv/bin/python scraper.py
+./venv/bin/python enrich_giant.py
+./venv/bin/python build_observed_products.py
+./venv/bin/python build_review_queue.py
+./venv/bin/python build_canonical_layer.py
+```
+
+## Inputs
+
+- Firefox cookies for `giantfood.com`
+- `GIANT_USER_ID` and `GIANT_LOYALTY_NUMBER` in `.env`, shell env, or prompts
+- Giant raw order payloads in `giant_output/raw/`
+
+## Outputs
+
+Current generated files live under `giant_output/`:
+
+- `orders.csv`: flattened visit/order rows from the Giant history API
+- `items.csv`: flattened raw line items from fetched order detail payloads
+- `items_enriched.csv`: deterministic parsed/enriched line items
+- `products_observed.csv`: retailer-facing observed product groups
+- `review_queue.csv`: products needing manual review
+- `products_canonical.csv`: shared canonical product rows
+- `product_links.csv`: observed-to-canonical links
+
+Raw json remains the source of truth:
+
+- `giant_output/raw/history.json`
+- `giant_output/raw/<order_id>.json`
+
+## Scripts
+
+- `scraper.py`: fetches Giant history/detail payloads and updates `orders.csv` and `items.csv`
+- `enrich_giant.py`: reads raw Giant order json and writes `items_enriched.csv`
+- `build_observed_products.py`: groups enriched rows into `products_observed.csv`
+- `build_review_queue.py`: generates `review_queue.csv` and preserves review status on reruns
+- `build_canonical_layer.py`: builds `products_canonical.csv` and `product_links.csv`
+
+## Notes on the current model
+
+- Observed products are retailer-specific: Giant, Costco.
+- Canonical products are the first cross-retailer layer.
+- Auto-linking is conservative:
+  exact UPC first, then exact normalized name plus exact size/unit context, then
+  exact normalized name when there is no size context to conflict.
+- Fee rows are excluded from auto-linking.
+- Unknown values are left blank instead of guessed.
+
+## Verification
+
+Run the test suite with:
+
+```bash
+./venv/bin/python -m unittest discover -s tests
+```
+
+Useful one-off rebuilds:
+
+```bash
+./venv/bin/python enrich_giant.py
+./venv/bin/python build_observed_products.py
+./venv/bin/python build_review_queue.py
+./venv/bin/python build_canonical_layer.py
+```
+
+## Project docs
+
+- `pm/tasks.org`: task log and evidence
+- `pm/data-model.org`: file layout and schema decisions
+
+## Status
+
+Completed through `t1.7`:
+
+- Giant receipt fetch CLI
+- data model and file layout
+- Giant parser/enricher
+- observed products
+- review queue
+- canonical layer scaffold
+- conservative auto-link rules
+
+Next planned task is `t1.8`: add a Costco raw ingest path.

build_canonical_layer.py

@@ -0,0 +1,212 @@
+import click
+
+from layer_helpers import read_csv_rows, representative_value, stable_id, write_csv_rows
+
+
+CANONICAL_FIELDS = [
+    "canonical_product_id",
+    "canonical_name",
+    "product_type",
+    "brand",
+    "variant",
+    "size_value",
+    "size_unit",
+    "pack_qty",
+    "measure_type",
+    "normalized_quantity",
+    "normalized_quantity_unit",
+    "notes",
+    "created_at",
+    "updated_at",
+]
+
+LINK_FIELDS = [
+    "observed_product_id",
+    "canonical_product_id",
+    "link_method",
+    "link_confidence",
+    "review_status",
+    "reviewed_by",
+    "reviewed_at",
+    "link_notes",
+]
+
+
+def to_float(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def normalized_quantity(row):
+    size_value = to_float(row.get("representative_size_value"))
+    pack_qty = to_float(row.get("representative_pack_qty")) or 1.0
+    size_unit = row.get("representative_size_unit", "")
+    measure_type = row.get("representative_measure_type", "")
+
+    if size_value is not None and size_unit:
+        return format(size_value * pack_qty, "g"), size_unit
+
+    if row.get("representative_pack_qty") and measure_type == "count":
+        return row["representative_pack_qty"], "count"
+
+    if measure_type == "each":
+        return "1", "each"
+
+    return "", ""
+
+
+def auto_link_rule(observed_row):
+    if observed_row.get("is_fee") == "true":
+        return "", "", ""
+
+    if observed_row.get("representative_upc"):
+        return (
+            "exact_upc",
+            f"upc={observed_row['representative_upc']}",
+            "high",
+        )
+
+    if (
+        observed_row.get("representative_name_norm")
+        and observed_row.get("representative_size_value")
+        and observed_row.get("representative_size_unit")
+    ):
+        return (
+            "exact_name_size",
+            "|".join(
+                [
+                    f"name={observed_row['representative_name_norm']}",
+                    f"size={observed_row['representative_size_value']}",
+                    f"unit={observed_row['representative_size_unit']}",
+                    f"pack={observed_row['representative_pack_qty']}",
+                    f"measure={observed_row['representative_measure_type']}",
+                ]
+            ),
+            "high",
+        )
+
+    if (
+        observed_row.get("representative_name_norm")
+        and not observed_row.get("representative_size_value")
+        and not observed_row.get("representative_size_unit")
+        and not observed_row.get("representative_pack_qty")
+    ):
+        return (
+            "exact_name",
+            "|".join(
+                [
+                    f"name={observed_row['representative_name_norm']}",
+                    f"measure={observed_row['representative_measure_type']}",
+                ]
+            ),
+            "medium",
+        )
+
+    return "", "", ""
+
+
+def canonical_row_for_group(canonical_product_id, group_rows, link_method):
+    quantity_value, quantity_unit = normalized_quantity(
+        {
+            "representative_size_value": representative_value(
+                group_rows, "representative_size_value"
+            ),
+            "representative_size_unit": representative_value(
+                group_rows, "representative_size_unit"
+            ),
+            "representative_pack_qty": representative_value(
+                group_rows, "representative_pack_qty"
+            ),
+            "representative_measure_type": representative_value(
+                group_rows, "representative_measure_type"
+            ),
+        }
+    )
+    return {
+        "canonical_product_id": canonical_product_id,
+        "canonical_name": representative_value(group_rows, "representative_name_norm"),
+        "product_type": "",
+        "brand": representative_value(group_rows, "representative_brand"),
+        "variant": representative_value(group_rows, "representative_variant"),
+        "size_value": representative_value(group_rows, "representative_size_value"),
+        "size_unit": representative_value(group_rows, "representative_size_unit"),
+        "pack_qty": representative_value(group_rows, "representative_pack_qty"),
+        "measure_type": representative_value(group_rows, "representative_measure_type"),
+        "normalized_quantity": quantity_value,
+        "normalized_quantity_unit": quantity_unit,
+        "notes": f"auto-linked via {link_method}",
+        "created_at": "",
+        "updated_at": "",
+    }
+
+
+def build_canonical_layer(observed_rows):
+    canonical_rows = []
+    link_rows = []
+    groups = {}
+
+    for observed_row in sorted(observed_rows, key=lambda row: row["observed_product_id"]):
+        link_method, group_key, confidence = auto_link_rule(observed_row)
+        if not group_key:
+            continue
+
+        canonical_product_id = stable_id("gcan", f"{link_method}|{group_key}")
+        groups.setdefault(canonical_product_id, {"method": link_method, "rows": []})
+        groups[canonical_product_id]["rows"].append(observed_row)
+        link_rows.append(
+            {
+                "observed_product_id": observed_row["observed_product_id"],
+                "canonical_product_id": canonical_product_id,
+                "link_method": link_method,
+                "link_confidence": confidence,
+                "review_status": "",
+                "reviewed_by": "",
+                "reviewed_at": "",
+                "link_notes": "",
+            }
+        )
+
+    for canonical_product_id, group in sorted(groups.items()):
+        canonical_rows.append(
+            canonical_row_for_group(
+                canonical_product_id, group["rows"], group["method"]
+            )
+        )
+
+    return canonical_rows, link_rows
+
+
+@click.command()
+@click.option(
+    "--observed-csv",
+    default="giant_output/products_observed.csv",
+    show_default=True,
+    help="Path to observed product rows.",
+)
+@click.option(
+    "--canonical-csv",
+    default="giant_output/products_canonical.csv",
+    show_default=True,
+    help="Path to canonical product output.",
+)
+@click.option(
+    "--links-csv",
+    default="giant_output/product_links.csv",
+    show_default=True,
+    help="Path to observed-to-canonical link output.",
+)
+def main(observed_csv, canonical_csv, links_csv):
+    observed_rows = read_csv_rows(observed_csv)
+    canonical_rows, link_rows = build_canonical_layer(observed_rows)
+    write_csv_rows(canonical_csv, canonical_rows, CANONICAL_FIELDS)
+    write_csv_rows(links_csv, link_rows, LINK_FIELDS)
+    click.echo(
+        f"wrote {len(canonical_rows)} canonical rows to {canonical_csv} and "
+        f"{len(link_rows)} links to {links_csv}"
+    )
+
+
+if __name__ == "__main__":
+    main()

build_observed_products.py

@@ -0,0 +1,147 @@
+from collections import defaultdict
+
+import click
+
+from layer_helpers import (
+    compact_join,
+    distinct_values,
+    first_nonblank,
+    read_csv_rows,
+    representative_value,
+    stable_id,
+    write_csv_rows,
+)
+
+
+OUTPUT_FIELDS = [
+    "observed_product_id",
+    "retailer",
+    "observed_key",
+    "representative_upc",
+    "representative_item_name",
+    "representative_name_norm",
+    "representative_brand",
+    "representative_variant",
+    "representative_size_value",
+    "representative_size_unit",
+    "representative_pack_qty",
+    "representative_measure_type",
+    "representative_image_url",
+    "is_store_brand",
+    "is_fee",
+    "first_seen_date",
+    "last_seen_date",
+    "times_seen",
+    "example_order_id",
+    "example_item_name",
+    "raw_name_examples",
+    "normalized_name_examples",
+    "example_prices",
+    "distinct_item_names_count",
+    "distinct_upcs_count",
+]
+
+
+def build_observed_key(row):
+    if row.get("upc"):
+        return "|".join(
+            [
+                row["retailer"],
+                f"upc={row['upc']}",
+                f"name={row['item_name_norm']}",
+            ]
+        )
+
+    return "|".join(
+        [
+            row["retailer"],
+            f"name={row['item_name_norm']}",
+            f"size={row['size_value']}",
+            f"unit={row['size_unit']}",
+            f"pack={row['pack_qty']}",
+            f"measure={row['measure_type']}",
+            f"store_brand={row['is_store_brand']}",
+            f"fee={row['is_fee']}",
+        ]
+    )
+
+
+def build_observed_products(rows):
+    grouped = defaultdict(list)
+    for row in rows:
+        grouped[build_observed_key(row)].append(row)
+
+    observed_rows = []
+    for observed_key, group_rows in sorted(grouped.items()):
+        ordered = sorted(
+            group_rows,
+            key=lambda row: (row["order_date"], row["order_id"], int(row["line_no"])),
+        )
+        observed_rows.append(
+            {
+                "observed_product_id": stable_id("gobs", observed_key),
+                "retailer": ordered[0]["retailer"],
+                "observed_key": observed_key,
+                "representative_upc": representative_value(ordered, "upc"),
+                "representative_item_name": representative_value(ordered, "item_name"),
+                "representative_name_norm": representative_value(
+                    ordered, "item_name_norm"
+                ),
+                "representative_brand": representative_value(ordered, "brand_guess"),
+                "representative_variant": representative_value(ordered, "variant"),
+                "representative_size_value": representative_value(ordered, "size_value"),
+                "representative_size_unit": representative_value(ordered, "size_unit"),
+                "representative_pack_qty": representative_value(ordered, "pack_qty"),
+                "representative_measure_type": representative_value(
+                    ordered, "measure_type"
+                ),
+                "representative_image_url": first_nonblank(ordered, "image_url"),
+                "is_store_brand": representative_value(ordered, "is_store_brand"),
+                "is_fee": representative_value(ordered, "is_fee"),
+                "first_seen_date": ordered[0]["order_date"],
+                "last_seen_date": ordered[-1]["order_date"],
+                "times_seen": str(len(ordered)),
+                "example_order_id": ordered[0]["order_id"],
+                "example_item_name": ordered[0]["item_name"],
+                "raw_name_examples": compact_join(
+                    distinct_values(ordered, "item_name"), limit=4
+                ),
+                "normalized_name_examples": compact_join(
+                    distinct_values(ordered, "item_name_norm"), limit=4
+                ),
+                "example_prices": compact_join(
+                    distinct_values(ordered, "line_total"), limit=4
+                ),
+                "distinct_item_names_count": str(
+                    len(distinct_values(ordered, "item_name"))
+                ),
+                "distinct_upcs_count": str(len(distinct_values(ordered, "upc"))),
+            }
+        )
+
+    observed_rows.sort(key=lambda row: row["observed_product_id"])
+    return observed_rows
+
+
+@click.command()
+@click.option(
+    "--items-enriched-csv",
+    default="giant_output/items_enriched.csv",
+    show_default=True,
+    help="Path to enriched Giant item rows.",
+)
+@click.option(
+    "--output-csv",
+    default="giant_output/products_observed.csv",
+    show_default=True,
+    help="Path to observed product output.",
+)
+def main(items_enriched_csv, output_csv):
+    rows = read_csv_rows(items_enriched_csv)
+    observed_rows = build_observed_products(rows)
+    write_csv_rows(output_csv, observed_rows, OUTPUT_FIELDS)
+    click.echo(f"wrote {len(observed_rows)} rows to {output_csv}")
+
+
+if __name__ == "__main__":
+    main()

build_review_queue.py

@@ -0,0 +1,168 @@
+from collections import defaultdict
+from datetime import date
+
+import click
+
+from layer_helpers import compact_join, distinct_values, read_csv_rows, stable_id, write_csv_rows
+
+
+OUTPUT_FIELDS = [
+    "review_id",
+    "queue_type",
+    "retailer",
+    "observed_product_id",
+    "canonical_product_id",
+    "reason_code",
+    "priority",
+    "raw_item_names",
+    "normalized_names",
+    "upc",
+    "image_url",
+    "example_prices",
+    "seen_count",
+    "status",
+    "resolution_notes",
+    "created_at",
+    "updated_at",
+]
+
+
+def existing_review_state(path):
+    try:
+        rows = read_csv_rows(path)
+    except FileNotFoundError:
+        return {}
+    return {row["review_id"]: row for row in rows}
+
+
+def review_reasons(observed_row):
+    reasons = []
+    if observed_row["is_fee"] == "true":
+        return reasons
+    if observed_row["distinct_upcs_count"] not in {"", "0", "1"}:
+        reasons.append(("multiple_upcs", "high"))
+    if observed_row["distinct_item_names_count"] not in {"", "0", "1"}:
+        reasons.append(("multiple_raw_names", "medium"))
+    if not observed_row["representative_image_url"]:
+        reasons.append(("missing_image", "medium"))
+    if not observed_row["representative_upc"]:
+        reasons.append(("missing_upc", "high"))
+    if not observed_row["representative_name_norm"]:
+        reasons.append(("missing_normalized_name", "high"))
+    return reasons
+
+
+def build_review_queue(observed_rows, item_rows, existing_rows, today_text):
+    by_observed = defaultdict(list)
+    for row in item_rows:
+        observed_id = row.get("observed_product_id", "")
+        if observed_id:
+            by_observed[observed_id].append(row)
+
+    queue_rows = []
+    for observed_row in observed_rows:
+        reasons = review_reasons(observed_row)
+        if not reasons:
+            continue
+
+        related_items = by_observed.get(observed_row["observed_product_id"], [])
+        raw_names = compact_join(distinct_values(related_items, "item_name"), limit=5)
+        norm_names = compact_join(
+            distinct_values(related_items, "item_name_norm"), limit=5
+        )
+        example_prices = compact_join(
+            distinct_values(related_items, "line_total"), limit=5
+        )
+
+        for reason_code, priority in reasons:
+            review_id = stable_id(
+                "rvw",
+                f"{observed_row['observed_product_id']}|{reason_code}",
+            )
+            prior = existing_rows.get(review_id, {})
+            queue_rows.append(
+                {
+                    "review_id": review_id,
+                    "queue_type": "observed_product",
+                    "retailer": observed_row["retailer"],
+                    "observed_product_id": observed_row["observed_product_id"],
+                    "canonical_product_id": prior.get("canonical_product_id", ""),
+                    "reason_code": reason_code,
+                    "priority": priority,
+                    "raw_item_names": raw_names,
+                    "normalized_names": norm_names,
+                    "upc": observed_row["representative_upc"],
+                    "image_url": observed_row["representative_image_url"],
+                    "example_prices": example_prices,
+                    "seen_count": observed_row["times_seen"],
+                    "status": prior.get("status", "pending"),
+                    "resolution_notes": prior.get("resolution_notes", ""),
+                    "created_at": prior.get("created_at", today_text),
+                    "updated_at": today_text,
+                }
+            )
+
+    queue_rows.sort(key=lambda row: (row["priority"], row["reason_code"], row["review_id"]))
+    return queue_rows
+
+
+def attach_observed_ids(item_rows, observed_rows):
+    observed_by_key = {row["observed_key"]: row["observed_product_id"] for row in observed_rows}
+    attached = []
+    for row in item_rows:
+        observed_key = "|".join(
+            [
+                row["retailer"],
+                f"upc={row['upc']}",
+                f"name={row['item_name_norm']}",
+            ]
+        ) if row.get("upc") else "|".join(
+            [
+                row["retailer"],
+                f"name={row['item_name_norm']}",
+                f"size={row['size_value']}",
+                f"unit={row['size_unit']}",
+                f"pack={row['pack_qty']}",
+                f"measure={row['measure_type']}",
+                f"store_brand={row['is_store_brand']}",
+                f"fee={row['is_fee']}",
+            ]
+        )
+        enriched = dict(row)
+        enriched["observed_product_id"] = observed_by_key.get(observed_key, "")
+        attached.append(enriched)
+    return attached
+
+
+@click.command()
+@click.option(
+    "--observed-csv",
+    default="giant_output/products_observed.csv",
+    show_default=True,
+    help="Path to observed product rows.",
+)
+@click.option(
+    "--items-enriched-csv",
+    default="giant_output/items_enriched.csv",
+    show_default=True,
+    help="Path to enriched Giant item rows.",
+)
+@click.option(
+    "--output-csv",
+    default="giant_output/review_queue.csv",
+    show_default=True,
+    help="Path to review queue output.",
+)
+def main(observed_csv, items_enriched_csv, output_csv):
+    observed_rows = read_csv_rows(observed_csv)
+    item_rows = read_csv_rows(items_enriched_csv)
+    item_rows = attach_observed_ids(item_rows, observed_rows)
+    existing_rows = existing_review_state(output_csv)
+    today_text = str(date.today())
+    queue_rows = build_review_queue(observed_rows, item_rows, existing_rows, today_text)
+    write_csv_rows(output_csv, queue_rows, OUTPUT_FIELDS)
+    click.echo(f"wrote {len(queue_rows)} rows to {output_csv}")
+
+
+if __name__ == "__main__":
+    main()

enrich_giant.py

@@ -0,0 +1,426 @@
+import csv
+import json
+import re
+from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
+from pathlib import Path
+
+import click
+
+
+PARSER_VERSION = "giant-enrich-v1"
+RETAILER = "giant"
+DEFAULT_INPUT_DIR = Path("giant_output/raw")
+DEFAULT_OUTPUT_CSV = Path("giant_output/items_enriched.csv")
+
+OUTPUT_FIELDS = [
+    "retailer",
+    "order_id",
+    "line_no",
+    "observed_item_key",
+    "order_date",
+    "pod_id",
+    "item_name",
+    "upc",
+    "category_id",
+    "category",
+    "qty",
+    "unit",
+    "unit_price",
+    "line_total",
+    "picked_weight",
+    "mvp_savings",
+    "reward_savings",
+    "coupon_savings",
+    "coupon_price",
+    "image_url",
+    "raw_order_path",
+    "item_name_norm",
+    "brand_guess",
+    "variant",
+    "size_value",
+    "size_unit",
+    "pack_qty",
+    "measure_type",
+    "is_store_brand",
+    "is_fee",
+    "price_per_each",
+    "price_per_lb",
+    "price_per_oz",
+    "parse_version",
+    "parse_notes",
+]
+
+STORE_BRAND_PREFIXES = {
+    "SB": "SB",
+    "NP": "NP",
+}
+
+ABBREVIATIONS = {
+    "APPLE": "APPLE",
+    "APPLES": "APPLES",
+    "APLE": "APPLE",
+    "BASIL": "BASIL",
+    "BLK": "BLACK",
+    "BNLS": "BONELESS",
+    "BRWN": "BROWN",
+    "CARROTS": "CARROTS",
+    "CHDR": "CHEDDAR",
+    "CHICKEN": "CHICKEN",
+    "CHOC": "CHOCOLATE",
+    "CHS": "CHEESE",
+    "CHSE": "CHEESE",
+    "CHZ": "CHEESE",
+    "CILANTRO": "CILANTRO",
+    "CKI": "COOKIE",
+    "CRSHD": "CRUSHED",
+    "FLR": "FLOUR",
+    "FRSH": "FRESH",
+    "GALA": "GALA",
+    "GRAHM": "GRAHAM",
+    "HOT": "HOT",
+    "HRSRDSH": "HORSERADISH",
+    "IMP": "IMPORTED",
+    "IQF": "IQF",
+    "LENTILS": "LENTILS",
+    "LG": "LARGE",
+    "MLK": "MILK",
+    "MSTRD": "MUSTARD",
+    "ONION": "ONION",
+    "ORG": "ORGANIC",
+    "PEPPER": "PEPPER",
+    "PEPPERS": "PEPPERS",
+    "POT": "POTATO",
+    "POTATO": "POTATO",
+    "PPR": "PEPPER",
+    "RICOTTA": "RICOTTA",
+    "ROASTER": "ROASTER",
+    "ROTINI": "ROTINI",
+    "SCE": "SAUCE",
+    "SLC": "SLICED",
+    "SPINCH": "SPINACH",
+    "SPNC": "SPINACH",
+    "SPINACH": "SPINACH",
+    "SQZ": "SQUEEZE",
+    "SWT": "SWEET",
+    "THYME": "THYME",
+    "TOM": "TOMATO",
+    "TOMS": "TOMATOES",
+    "TRTL": "TORTILLA",
+    "VEG": "VEGETABLE",
+    "VINEGAR": "VINEGAR",
+    "WHT": "WHITE",
+    "WHOLE": "WHOLE",
+    "YLW": "YELLOW",
+    "YLWGLD": "YELLOW_GOLD",
+}
+
+FEE_PATTERNS = [
+    re.compile(r"\bBAG CHARGE\b"),
+    re.compile(r"\bDISC AT TOTAL\b"),
+]
+
+SIZE_RE = re.compile(r"(?<![A-Z0-9])(\d+(?:\.\d+)?)(?:\s*)(OZ|Z|LB|LBS|ML|L|FZ|FL OZ|QT|PT|GAL|GA)\b")
+PACK_RE = re.compile(r"(?<![A-Z0-9])(\d+(?:\.\d+)?)(?:\s*)(CT|PK|PKG|PACK)\b")
+
+
+def to_decimal(value):
+    if value in ("", None):
+        return None
+
+    try:
+        return Decimal(str(value))
+    except (InvalidOperation, ValueError):
+        return None
+
+
+def format_decimal(value, places=4):
+    if value is None:
+        return ""
+
+    quant = Decimal("1").scaleb(-places)
+    normalized = value.quantize(quant, rounding=ROUND_HALF_UP).normalize()
+    return format(normalized, "f")
+
+
+def normalize_whitespace(value):
+    return " ".join(str(value or "").strip().split())
+
+
+def clean_item_name(name):
+    cleaned = normalize_whitespace(name).upper()
+    cleaned = re.sub(r"^\+", "", cleaned)
+    cleaned = re.sub(r"^PLU#\d+\s*", "", cleaned)
+    cleaned = cleaned.replace("#", " ")
+    return normalize_whitespace(cleaned)
+
+
+def extract_store_brand_prefix(cleaned_name):
+    for prefix, brand in STORE_BRAND_PREFIXES.items():
+        if cleaned_name == prefix or cleaned_name.startswith(f"{prefix} "):
+            return prefix, brand
+    return "", ""
+
+
+def extract_image_url(item):
+    image = item.get("image")
+    if isinstance(image, dict):
+        for key in ["xlarge", "large", "medium", "small"]:
+            value = image.get(key)
+            if value:
+                return value
+    if isinstance(image, str):
+        return image
+    return ""
+
+
+def parse_size_and_pack(cleaned_name):
+    size_value = ""
+    size_unit = ""
+    pack_qty = ""
+
+    size_matches = list(SIZE_RE.finditer(cleaned_name))
+    if size_matches:
+        match = size_matches[-1]
+        size_value = normalize_number(match.group(1))
+        size_unit = normalize_unit(match.group(2))
+
+    pack_matches = list(PACK_RE.finditer(cleaned_name))
+    if pack_matches:
+        match = pack_matches[-1]
+        pack_qty = normalize_number(match.group(1))
+
+    return size_value, size_unit, pack_qty
+
+
+def normalize_number(value):
+    decimal = to_decimal(value)
+    if decimal is None:
+        return ""
+    return format(decimal.normalize(), "f")
+
+
+def normalize_unit(unit):
+    collapsed = normalize_whitespace(unit).upper()
+    return {
+        "Z": "oz",
+        "OZ": "oz",
+        "FZ": "fl_oz",
+        "FL OZ": "fl_oz",
+        "LB": "lb",
+        "LBS": "lb",
+        "ML": "ml",
+        "L": "l",
+        "QT": "qt",
+        "PT": "pt",
+        "GAL": "gal",
+        "GA": "gal",
+    }.get(collapsed, collapsed.lower())
+
+
+def strip_measure_tokens(cleaned_name):
+    without_sizes = SIZE_RE.sub(" ", cleaned_name)
+    without_measures = PACK_RE.sub(" ", without_sizes)
+    return normalize_whitespace(without_measures)
+
+
+def expand_token(token):
+    return ABBREVIATIONS.get(token, token)
+
+
+def normalize_item_name(cleaned_name):
+    prefix, _brand = extract_store_brand_prefix(cleaned_name)
+    base = cleaned_name
+    if prefix:
+        base = normalize_whitespace(base[len(prefix):])
+
+    base = strip_measure_tokens(base)
+    expanded_tokens = [expand_token(token) for token in base.split()]
+    expanded = " ".join(token for token in expanded_tokens if token)
+    return normalize_whitespace(expanded)
+
+
+def guess_measure_type(item, size_unit, pack_qty):
+    unit = normalize_whitespace(item.get("lbEachCd")).upper()
+    picked_weight = to_decimal(item.get("totalPickedWeight"))
+    qty = to_decimal(item.get("shipQy"))
+
+    if unit == "LB" or (picked_weight is not None and picked_weight > 0 and unit != "EA"):
+        return "weight"
+    if size_unit in {"lb", "oz"}:
+        return "weight"
+    if size_unit in {"ml", "l", "qt", "pt", "gal", "fl_oz"}:
+        return "volume"
+    if pack_qty:
+        return "count"
+    if unit == "EA" or (qty is not None and qty > 0):
+        return "each"
+    return ""
+
+
+def is_fee_item(cleaned_name):
+    return any(pattern.search(cleaned_name) for pattern in FEE_PATTERNS)
+
+
+def derive_prices(item, measure_type, size_value="", size_unit="", pack_qty=""):
+    qty = to_decimal(item.get("shipQy"))
+    line_total = to_decimal(item.get("groceryAmount"))
+    picked_weight = to_decimal(item.get("totalPickedWeight"))
+    parsed_size = to_decimal(size_value)
+    parsed_pack = to_decimal(pack_qty) or Decimal("1")
+
+    price_per_each = ""
+    price_per_lb = ""
+    price_per_oz = ""
+
+    if line_total is None:
+        return price_per_each, price_per_lb, price_per_oz
+
+    if measure_type == "each" and qty not in (None, Decimal("0")):
+        price_per_each = format_decimal(line_total / qty)
+
+    if measure_type == "count" and qty not in (None, Decimal("0")):
+        price_per_each = format_decimal(line_total / qty)
+
+    if measure_type == "weight" and picked_weight not in (None, Decimal("0")):
+        per_lb = line_total / picked_weight
+        price_per_lb = format_decimal(per_lb)
+        price_per_oz = format_decimal(per_lb / Decimal("16"))
+        return price_per_each, price_per_lb, price_per_oz
+
+    if measure_type == "weight" and parsed_size not in (None, Decimal("0")) and qty not in (None, Decimal("0")):
+        total_units = qty * parsed_pack * parsed_size
+        if size_unit == "lb":
+            per_lb = line_total / total_units
+            price_per_lb = format_decimal(per_lb)
+            price_per_oz = format_decimal(per_lb / Decimal("16"))
+        elif size_unit == "oz":
+            per_oz = line_total / total_units
+            price_per_oz = format_decimal(per_oz)
+            price_per_lb = format_decimal(per_oz * Decimal("16"))
+
+    return price_per_each, price_per_lb, price_per_oz
+
+
+def parse_item(order_id, order_date, raw_path, line_no, item):
+    cleaned_name = clean_item_name(item.get("itemName", ""))
+    size_value, size_unit, pack_qty = parse_size_and_pack(cleaned_name)
+    prefix, brand_guess = extract_store_brand_prefix(cleaned_name)
+    normalized_name = normalize_item_name(cleaned_name)
+    measure_type = guess_measure_type(item, size_unit, pack_qty)
+    price_per_each, price_per_lb, price_per_oz = derive_prices(
+        item,
+        measure_type,
+        size_value=size_value,
+        size_unit=size_unit,
+        pack_qty=pack_qty,
+    )
+    is_fee = is_fee_item(cleaned_name)
+    parse_notes = []
+
+    if prefix:
+        parse_notes.append(f"store_brand_prefix={prefix}")
+    if is_fee:
+        parse_notes.append("fee_item")
+    if size_value and not size_unit:
+        parse_notes.append("size_without_unit")
+
+    return {
+        "retailer": RETAILER,
+        "order_id": str(order_id),
+        "line_no": str(line_no),
+        "observed_item_key": f"{RETAILER}:{order_id}:{line_no}",
+        "order_date": normalize_whitespace(order_date),
+        "pod_id": stringify(item.get("podId")),
+        "item_name": stringify(item.get("itemName")),
+        "upc": stringify(item.get("primUpcCd")),
+        "category_id": stringify(item.get("categoryId")),
+        "category": stringify(item.get("categoryDesc")),
+        "qty": stringify(item.get("shipQy")),
+        "unit": stringify(item.get("lbEachCd")),
+        "unit_price": stringify(item.get("unitPrice")),
+        "line_total": stringify(item.get("groceryAmount")),
+        "picked_weight": stringify(item.get("totalPickedWeight")),
+        "mvp_savings": stringify(item.get("mvpSavings")),
+        "reward_savings": stringify(item.get("rewardSavings")),
+        "coupon_savings": stringify(item.get("couponSavings")),
+        "coupon_price": stringify(item.get("couponPrice")),
+        "image_url": extract_image_url(item),
+        "raw_order_path": raw_path.as_posix(),
+        "item_name_norm": normalized_name,
+        "brand_guess": brand_guess,
+        "variant": "",
+        "size_value": size_value,
+        "size_unit": size_unit,
+        "pack_qty": pack_qty,
+        "measure_type": measure_type,
+        "is_store_brand": "true" if bool(prefix) else "false",
+        "is_fee": "true" if is_fee else "false",
+        "price_per_each": price_per_each,
+        "price_per_lb": price_per_lb,
+        "price_per_oz": price_per_oz,
+        "parse_version": PARSER_VERSION,
+        "parse_notes": ";".join(parse_notes),
+    }
+
+
+def stringify(value):
+    if value is None:
+        return ""
+    return str(value)
+
+
+def iter_order_rows(raw_dir):
+    for path in sorted(raw_dir.glob("*.json")):
+        if path.name == "history.json":
+            continue
+
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        order_id = payload.get("orderId", path.stem)
+        order_date = payload.get("orderDate", "")
+
+        for line_no, item in enumerate(payload.get("items", []), start=1):
+            yield parse_item(order_id, order_date, path, line_no, item)
+
+
+def build_items_enriched(raw_dir):
+    rows = list(iter_order_rows(raw_dir))
+    rows.sort(key=lambda row: (row["order_date"], row["order_id"], int(row["line_no"])))
+    return rows
+
+
+def write_csv(path, rows):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=OUTPUT_FIELDS)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+@click.command()
+@click.option(
+    "--input-dir",
+    default=str(DEFAULT_INPUT_DIR),
+    show_default=True,
+    help="Directory containing Giant raw order json files.",
+)
+@click.option(
+    "--output-csv",
+    default=str(DEFAULT_OUTPUT_CSV),
+    show_default=True,
+    help="CSV path for enriched Giant item rows.",
+)
+def main(input_dir, output_csv):
+    raw_dir = Path(input_dir)
+    output_path = Path(output_csv)
+
+    if not raw_dir.exists():
+        raise click.ClickException(f"input dir does not exist: {raw_dir}")
+
+    rows = build_items_enriched(raw_dir)
+    write_csv(output_path, rows)
+
+    click.echo(f"wrote {len(rows)} rows to {output_path}")
+
+
+if __name__ == "__main__":
+    main()

layer_helpers.py

@@ -0,0 +1,54 @@
+import csv
+import hashlib
+from collections import Counter
+from pathlib import Path
+
+
+def read_csv_rows(path):
+    path = Path(path)
+    with path.open(newline="", encoding="utf-8") as handle:
+        return list(csv.DictReader(handle))
+
+
+def write_csv_rows(path, rows, fieldnames):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def stable_id(prefix, raw_key):
+    digest = hashlib.sha1(str(raw_key).encode("utf-8")).hexdigest()[:12]
+    return f"{prefix}_{digest}"
+
+
+def first_nonblank(rows, field):
+    for row in rows:
+        value = row.get(field, "")
+        if value:
+            return value
+    return ""
+
+
+def representative_value(rows, field):
+    values = [row.get(field, "") for row in rows if row.get(field, "")]
+    if not values:
+        return ""
+    counts = Counter(values)
+    return sorted(counts.items(), key=lambda item: (-item[1], item[0]))[0][0]
+
+
+def distinct_values(rows, field):
+    return sorted({row.get(field, "") for row in rows if row.get(field, "")})
+
+
+def compact_join(values, limit=3):
+    unique = []
+    seen = set()
+    for value in values:
+        if value and value not in seen:
+            seen.add(value)
+            unique.append(value)
+    return " | ".join(unique[:limit])

pm/tasks.org

@@ -32,11 +32,11 @@
 - keep schema minimal but extensible
 
 ** evidence
-- commit:
+- commit: `42dbae1` on branch `cx`
 - tests: reviewed `giant_output/raw/history.json`, one sample raw order json, `giant_output/orders.csv`, `giant_output/items.csv`; documented schemas in `pm/data-model.org`
 - date: 2026-03-15
 
-* [ ] t1.3: build giant parser/enricher from raw json (2-4 commits)
+* [X] t1.3: build giant parser/enricher from raw json (2-4 commits)
 ** acceptance criteria
 - parser reads giant raw order json files
 - outputs `items_enriched.csv`
@@ -54,11 +54,11 @@
 - parser should preserve ambiguity rather than hallucinating precision
 
 ** evidence
-- commit:
+- commit: `14f2cc2` on branch `cx`
-- tests:
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python enrich_giant.py`; verified `giant_output/items_enriched.csv` on real raw data
-- date:
+- date: 2026-03-16
 
-* [ ] t1.4: generate observed-product layer from enriched items (2-3 commits)
+* [X] t1.4: generate observed-product layer from enriched items (2-3 commits)
 
 ** acceptance criteria
 - distinct observed products are generated from enriched giant items
@@ -76,11 +76,11 @@
 - likely key is some combo of retailer + upc + normalized name
 
 ** evidence
-- commit:
+- commit: `dc39214` on branch `cx`
-- tests:
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python build_observed_products.py`; verified `giant_output/products_observed.csv`
-- date:
+- date: 2026-03-16
 
-* [ ] t1.5: build review queue for unresolved or low-confidence products (1-3 commits)
+* [X] t1.5: build review queue for unresolved or low-confidence products (1-3 commits)
 
 ** acceptance criteria
 - produce a review file containing observed products needing manual review
@@ -98,11 +98,11 @@
 - optimize for “approve once, remember forever”
 
 ** evidence
-- commit:
+- commit: `9b13ec3` on branch `cx`
-- tests:
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python build_review_queue.py`; verified `giant_output/review_queue.csv`
-- date:
+- date: 2026-03-16
 
-* [ ] t1.6: create canonical product layer and observed→canonical links (2-4 commits)
+* [X] t1.6: create canonical product layer and observed→canonical links (2-4 commits)
 
 ** acceptance criteria
 - define and create `products_canonical.csv`
@@ -120,11 +120,11 @@
 - do not require llm assistance for v1
 
 ** evidence
-- commit:
+- commit: `347cd44` on branch `cx`
-- tests:
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python build_canonical_layer.py`; verified seeded `giant_output/products_canonical.csv` and `giant_output/product_links.csv`
-- date:
+- date: 2026-03-16
 
-* [ ] t1.7: implement auto-link rules for easy matches (2-3 commits)
+* [X] t1.7: implement auto-link rules for easy matches (2-3 commits)
 
 ** acceptance criteria
 - auto-link can match observed products to canonical products using deterministic rules
@@ -139,43 +139,104 @@
 - false positives are worse than unresolved items
 
 ** evidence
-- commit:
+- commit: `385a31c` on branch `cx`
-- tests:
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python build_canonical_layer.py`; verified auto-linked `giant_output/products_canonical.csv` and `giant_output/product_links.csv`
-- date:
+- date: 2026-03-16
 
 * [ ] t1.8: support costco raw ingest path (2-5 commits)
 
 ** acceptance criteria
 - add a costco-specific raw ingest/export path
-- output costco line items into the same shared raw/enriched schema family
+- fetch costco receipt summary and receipt detail payloads from graphql endpoint
-- confirm at least one product class can exist as:
+- persist raw json under `costco_output/raw/orders.csv` and `./items.csv`, same format as giant
-  - giant observed product
+- costco-native identifiers such as `transactionBarcode` as order id and `itemNumber` as retailer item id
-  - costco observed product
+- preserve discount/coupon rows rather than dropping
-  - one shared canonical product
 
 ** notes
-- this is the proof that the architecture generalizes
+- focus on raw costco acquisistion and flattening
-- don’t chase perfection before the second retailer lands
+- do not force costco identifiers into `upc`
+- bearer/auth values should come from local env, not source
 
 ** evidence
 - commit:
 - tests:
 - date:
 
-* [ ] t1.9: compute normalized comparison metrics (2-3 commits)
+* [ ] t1.8.1: support costco parser/enricher path (2-4 commits)
 
 ** acceptance criteria
-- derive normalized comparison fields where possible:
+- add a costco-specific enrich step producing `costco_output/items_enriched.csv`
-  - price per lb
+- output rows into the same shared enriched schema family as Giant
-  - price per oz
+- support costco-specific parsing for:
-  - price per each
+  - `itemDescription01` + `itemDescription02`
-  - price per count
+  - `itemNumber` as `retailer_item_id`
-- metrics are attached at canonical or linked-observed level as appropriate
+  - discount lines / negative rows
-- emit obvious nulls when basis is unknown rather than inventing values
+  - common size patterns such as `25#`, `48 OZ`, `2/24 OZ`, `6-PACK`
+- preserve obvious unknowns as blank rather than guessed values
 
 ** notes
-- this is where “gala apples 5 lb bag vs other gala apples” becomes possible
+- this is the real schema compatibility proof, not raw ingest alone
-- units discipline matters a lot here
+- expect weaker identifiers than Giant
+
+** evidence
+- commit:
+- tests:
+- date:
+* [ ] t1.8.2: validate cross-retailer observed/canonical flow (1-3 commits)
+
+** acceptance criteria
+- feed Giant and Costco enriched rows through the same observed/canonical pipeline
+- confirm at least one product class can exist as:
+  - Giant observed product
+  - Costco observed product
+  - one shared canonical product
+- document the exact example used for proof
+
+** notes
+- keep this to one or two well-behaved product classes first
+- apples, eggs, bananas, or flour are better than weird prepared foods
+
+** evidence
+- commit:
+- tests:
+- date:
+* [ ] t1.8.3: extend shared schema for retailer-native ids and adjustment lines (1-2 commits)
+
+** acceptance criteria
+- add shared fields needed for non-upc retailers, including:
+  - `retailer_item_id`
+  - `is_discount_line`
+  - `is_coupon_line` or equivalent if needed
+- keep `upc` nullable across the pipeline
+- update downstream builders/tests to accept retailers with blank `upc`
+
+** notes
+- this prevents costco from becoming a schema hack
+- do this once instead of sprinkling exceptions everywhere
+
+** evidence
+- commit:
+- tests:
+- date:
+* [ ] t1.9: compute normalized comparison metrics (2-4 commits)
+
+** acceptance criteria
+- derive normalized comparison fields where possible on enriched or observed product rows:
+  - `price_per_lb`
+  - `price_per_oz`
+  - `price_per_each`
+  - `price_per_count`
+- preserve the source basis used to derive each metric, e.g.:
+  - parsed size/unit
+  - receipt weight
+  - explicit count/pack
+- emit nulls when basis is unknown, conflicting, or ambiguous
+- document at least one Giant vs Costco comparison example using the normalized metrics
+
+** notes
+- compute metrics as close to the raw observation as possible
+- canonical layer can aggregate later, but should not invent missing unit economics
+- unit discipline matters more than coverage
 
 ** evidence
 - commit:

tests/test_canonical_layer.py

@@ -0,0 +1,84 @@
+import unittest
+
+import build_canonical_layer
+
+
+class CanonicalLayerTests(unittest.TestCase):
+    def test_build_canonical_layer_auto_links_exact_upc_and_name_size(self):
+        observed_rows = [
+            {
+                "observed_product_id": "gobs_1",
+                "representative_upc": "111",
+                "representative_name_norm": "GALA APPLE",
+                "representative_brand": "SB",
+                "representative_variant": "",
+                "representative_size_value": "5",
+                "representative_size_unit": "lb",
+                "representative_pack_qty": "",
+                "representative_measure_type": "weight",
+                "is_fee": "false",
+            },
+            {
+                "observed_product_id": "gobs_2",
+                "representative_upc": "111",
+                "representative_name_norm": "LARGE WHITE EGGS",
+                "representative_brand": "SB",
+                "representative_variant": "",
+                "representative_size_value": "",
+                "representative_size_unit": "",
+                "representative_pack_qty": "18",
+                "representative_measure_type": "count",
+                "is_fee": "false",
+            },
+            {
+                "observed_product_id": "gobs_3",
+                "representative_upc": "",
+                "representative_name_norm": "ROTINI",
+                "representative_brand": "",
+                "representative_variant": "",
+                "representative_size_value": "16",
+                "representative_size_unit": "oz",
+                "representative_pack_qty": "",
+                "representative_measure_type": "weight",
+                "is_fee": "false",
+            },
+            {
+                "observed_product_id": "gobs_4",
+                "representative_upc": "",
+                "representative_name_norm": "ROTINI",
+                "representative_brand": "SB",
+                "representative_variant": "",
+                "representative_size_value": "16",
+                "representative_size_unit": "oz",
+                "representative_pack_qty": "",
+                "representative_measure_type": "weight",
+                "is_fee": "false",
+            },
+            {
+                "observed_product_id": "gobs_5",
+                "representative_upc": "",
+                "representative_name_norm": "GL BAG CHARGE",
+                "representative_brand": "",
+                "representative_variant": "",
+                "representative_size_value": "",
+                "representative_size_unit": "",
+                "representative_pack_qty": "",
+                "representative_measure_type": "each",
+                "is_fee": "true",
+            },
+        ]
+
+        canonicals, links = build_canonical_layer.build_canonical_layer(observed_rows)
+
+        self.assertEqual(2, len(canonicals))
+        self.assertEqual(4, len(links))
+        methods = {row["observed_product_id"]: row["link_method"] for row in links}
+        self.assertEqual("exact_upc", methods["gobs_1"])
+        self.assertEqual("exact_upc", methods["gobs_2"])
+        self.assertEqual("exact_name_size", methods["gobs_3"])
+        self.assertEqual("exact_name_size", methods["gobs_4"])
+        self.assertNotIn("gobs_5", methods)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_enrich_giant.py

@@ -0,0 +1,190 @@
+import csv
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+import enrich_giant
+
+
+class EnrichGiantTests(unittest.TestCase):
+    def test_parse_size_and_pack_handles_pack_and_weight_tokens(self):
+        size_value, size_unit, pack_qty = enrich_giant.parse_size_and_pack(
+            "COKE CHERRY 6PK 7.5Z"
+        )
+
+        self.assertEqual("7.5", size_value)
+        self.assertEqual("oz", size_unit)
+        self.assertEqual("6", pack_qty)
+
+    def test_parse_item_marks_store_brand_fee_and_weight_prices(self):
+        row = enrich_giant.parse_item(
+            order_id="abc123",
+            order_date="2026-03-01",
+            raw_path=Path("raw/abc123.json"),
+            line_no=1,
+            item={
+                "podId": 1,
+                "shipQy": 1,
+                "totalPickedWeight": 2,
+                "unitPrice": 3.98,
+                "itemName": "+SB GALA APPLE 5 LB",
+                "lbEachCd": "LB",
+                "groceryAmount": 3.98,
+                "primUpcCd": "111",
+                "mvpSavings": 0,
+                "rewardSavings": 0,
+                "couponSavings": 0,
+                "couponPrice": 0,
+                "categoryId": "1",
+                "categoryDesc": "Grocery",
+                "image": {"large": "https://example.test/apple.jpg"},
+            },
+        )
+
+        self.assertEqual("SB", row["brand_guess"])
+        self.assertEqual("GALA APPLE", row["item_name_norm"])
+        self.assertEqual("5", row["size_value"])
+        self.assertEqual("lb", row["size_unit"])
+        self.assertEqual("weight", row["measure_type"])
+        self.assertEqual("true", row["is_store_brand"])
+        self.assertEqual("1.99", row["price_per_lb"])
+        self.assertEqual("0.1244", row["price_per_oz"])
+        self.assertEqual("https://example.test/apple.jpg", row["image_url"])
+
+        fee_row = enrich_giant.parse_item(
+            order_id="abc123",
+            order_date="2026-03-01",
+            raw_path=Path("raw/abc123.json"),
+            line_no=2,
+            item={
+                "podId": 2,
+                "shipQy": 1,
+                "totalPickedWeight": 0,
+                "unitPrice": 0.05,
+                "itemName": "GL BAG CHARGE",
+                "lbEachCd": "EA",
+                "groceryAmount": 0.05,
+                "primUpcCd": "",
+                "mvpSavings": 0,
+                "rewardSavings": 0,
+                "couponSavings": 0,
+                "couponPrice": 0,
+                "categoryId": "1",
+                "categoryDesc": "Grocery",
+            },
+        )
+
+        self.assertEqual("true", fee_row["is_fee"])
+        self.assertEqual("GL BAG CHARGE", fee_row["item_name_norm"])
+
+    def test_parse_item_derives_packaged_weight_prices_from_size_tokens(self):
+        row = enrich_giant.parse_item(
+            order_id="abc123",
+            order_date="2026-03-01",
+            raw_path=Path("raw/abc123.json"),
+            line_no=1,
+            item={
+                "podId": 1,
+                "shipQy": 2,
+                "totalPickedWeight": 0,
+                "unitPrice": 3.0,
+                "itemName": "PEPSI 6PK 7.5Z",
+                "lbEachCd": "EA",
+                "groceryAmount": 6.0,
+                "primUpcCd": "111",
+                "mvpSavings": 0,
+                "rewardSavings": 0,
+                "couponSavings": 0,
+                "couponPrice": 0,
+                "categoryId": "1",
+                "categoryDesc": "Grocery",
+            },
+        )
+
+        self.assertEqual("weight", row["measure_type"])
+        self.assertEqual("6", row["pack_qty"])
+        self.assertEqual("7.5", row["size_value"])
+        self.assertEqual("0.0667", row["price_per_oz"])
+        self.assertEqual("1.0667", row["price_per_lb"])
+
+    def test_build_items_enriched_reads_raw_order_files_and_writes_csv(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            raw_dir = Path(tmpdir) / "raw"
+            raw_dir.mkdir()
+            (raw_dir / "history.json").write_text("{}", encoding="utf-8")
+            (raw_dir / "order-2.json").write_text(
+                json.dumps(
+                    {
+                        "orderId": "order-2",
+                        "orderDate": "2026-03-02",
+                        "items": [
+                            {
+                                "podId": 20,
+                                "shipQy": 1,
+                                "totalPickedWeight": 0,
+                                "unitPrice": 2.99,
+                                "itemName": "SB ROTINI 16Z",
+                                "lbEachCd": "EA",
+                                "groceryAmount": 2.99,
+                                "primUpcCd": "222",
+                                "mvpSavings": 0,
+                                "rewardSavings": 0,
+                                "couponSavings": 0,
+                                "couponPrice": 0,
+                                "categoryId": "1",
+                                "categoryDesc": "Grocery",
+                                "image": {"small": "https://example.test/rotini.jpg"},
+                            }
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            (raw_dir / "order-1.json").write_text(
+                json.dumps(
+                    {
+                        "orderId": "order-1",
+                        "orderDate": "2026-03-01",
+                        "items": [
+                            {
+                                "podId": 10,
+                                "shipQy": 2,
+                                "totalPickedWeight": 0,
+                                "unitPrice": 1.5,
+                                "itemName": "PEPSI 6PK 7.5Z",
+                                "lbEachCd": "EA",
+                                "groceryAmount": 3.0,
+                                "primUpcCd": "111",
+                                "mvpSavings": 0,
+                                "rewardSavings": 0,
+                                "couponSavings": 0,
+                                "couponPrice": 0,
+                                "categoryId": "1",
+                                "categoryDesc": "Grocery",
+                            }
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            rows = enrich_giant.build_items_enriched(raw_dir)
+            output_csv = Path(tmpdir) / "items_enriched.csv"
+            enrich_giant.write_csv(output_csv, rows)
+
+            self.assertEqual(["order-1", "order-2"], [row["order_id"] for row in rows])
+            self.assertEqual("PEPSI", rows[0]["item_name_norm"])
+            self.assertEqual("6", rows[0]["pack_qty"])
+            self.assertEqual("7.5", rows[0]["size_value"])
+            self.assertEqual("true", rows[1]["is_store_brand"])
+
+            with output_csv.open(newline="", encoding="utf-8") as handle:
+                written_rows = list(csv.DictReader(handle))
+
+            self.assertEqual(2, len(written_rows))
+            self.assertEqual(enrich_giant.OUTPUT_FIELDS, list(written_rows[0].keys()))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_observed_products.py

@@ -0,0 +1,60 @@
+import unittest
+
+import build_observed_products
+
+
+class ObservedProductTests(unittest.TestCase):
+    def test_build_observed_products_aggregates_rows_with_same_key(self):
+        rows = [
+            {
+                "retailer": "giant",
+                "order_id": "1",
+                "line_no": "1",
+                "order_date": "2026-01-01",
+                "item_name": "SB GALA APPLE 5LB",
+                "item_name_norm": "GALA APPLE",
+                "upc": "111",
+                "brand_guess": "SB",
+                "variant": "",
+                "size_value": "5",
+                "size_unit": "lb",
+                "pack_qty": "",
+                "measure_type": "weight",
+                "image_url": "https://example.test/a.jpg",
+                "is_store_brand": "true",
+                "is_fee": "false",
+                "line_total": "7.99",
+            },
+            {
+                "retailer": "giant",
+                "order_id": "2",
+                "line_no": "1",
+                "order_date": "2026-01-10",
+                "item_name": "SB GALA APPLE 5 LB",
+                "item_name_norm": "GALA APPLE",
+                "upc": "111",
+                "brand_guess": "SB",
+                "variant": "",
+                "size_value": "5",
+                "size_unit": "lb",
+                "pack_qty": "",
+                "measure_type": "weight",
+                "image_url": "",
+                "is_store_brand": "true",
+                "is_fee": "false",
+                "line_total": "8.49",
+            },
+        ]
+
+        observed = build_observed_products.build_observed_products(rows)
+
+        self.assertEqual(1, len(observed))
+        self.assertEqual("2", observed[0]["times_seen"])
+        self.assertEqual("2026-01-01", observed[0]["first_seen_date"])
+        self.assertEqual("2026-01-10", observed[0]["last_seen_date"])
+        self.assertEqual("111", observed[0]["representative_upc"])
+        self.assertIn("SB GALA APPLE 5LB", observed[0]["raw_name_examples"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_review_queue.py

@@ -0,0 +1,124 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+import build_observed_products
+import build_review_queue
+from layer_helpers import write_csv_rows
+
+
+class ReviewQueueTests(unittest.TestCase):
+    def test_build_review_queue_preserves_existing_status(self):
+        observed_rows = [
+            {
+                "observed_product_id": "gobs_1",
+                "retailer": "giant",
+                "representative_upc": "111",
+                "representative_image_url": "",
+                "representative_name_norm": "GALA APPLE",
+                "times_seen": "2",
+                "distinct_item_names_count": "2",
+                "distinct_upcs_count": "1",
+                "is_fee": "false",
+            }
+        ]
+        item_rows = [
+            {
+                "observed_product_id": "gobs_1",
+                "item_name": "SB GALA APPLE 5LB",
+                "item_name_norm": "GALA APPLE",
+                "line_total": "7.99",
+            },
+            {
+                "observed_product_id": "gobs_1",
+                "item_name": "SB GALA APPLE 5 LB",
+                "item_name_norm": "GALA APPLE",
+                "line_total": "8.49",
+            },
+        ]
+        existing = {
+            build_review_queue.stable_id("rvw", "gobs_1|missing_image"): {
+                "status": "approved",
+                "resolution_notes": "looked fine",
+                "created_at": "2026-03-15",
+            }
+        }
+
+        queue = build_review_queue.build_review_queue(
+            observed_rows, item_rows, existing, "2026-03-16"
+        )
+
+        self.assertEqual(2, len(queue))
+        missing_image = [row for row in queue if row["reason_code"] == "missing_image"][0]
+        self.assertEqual("approved", missing_image["status"])
+        self.assertEqual("looked fine", missing_image["resolution_notes"])
+
+    def test_review_queue_main_writes_output(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            observed_path = Path(tmpdir) / "products_observed.csv"
+            items_path = Path(tmpdir) / "items_enriched.csv"
+            output_path = Path(tmpdir) / "review_queue.csv"
+
+            observed_rows = [
+                {
+                    "observed_product_id": "gobs_1",
+                    "retailer": "giant",
+                    "observed_key": "giant|upc=111|name=GALA APPLE",
+                    "representative_upc": "111",
+                    "representative_item_name": "SB GALA APPLE 5LB",
+                    "representative_name_norm": "GALA APPLE",
+                    "representative_brand": "SB",
+                    "representative_variant": "",
+                    "representative_size_value": "5",
+                    "representative_size_unit": "lb",
+                    "representative_pack_qty": "",
+                    "representative_measure_type": "weight",
+                    "representative_image_url": "",
+                    "is_store_brand": "true",
+                    "is_fee": "false",
+                    "first_seen_date": "2026-01-01",
+                    "last_seen_date": "2026-01-10",
+                    "times_seen": "2",
+                    "example_order_id": "1",
+                    "example_item_name": "SB GALA APPLE 5LB",
+                    "raw_name_examples": "SB GALA APPLE 5LB | SB GALA APPLE 5 LB",
+                    "normalized_name_examples": "GALA APPLE",
+                    "example_prices": "7.99 | 8.49",
+                    "distinct_item_names_count": "2",
+                    "distinct_upcs_count": "1",
+                }
+            ]
+            item_rows = [
+                {
+                    "retailer": "giant",
+                    "order_id": "1",
+                    "line_no": "1",
+                    "item_name": "SB GALA APPLE 5LB",
+                    "item_name_norm": "GALA APPLE",
+                    "upc": "111",
+                    "size_value": "5",
+                    "size_unit": "lb",
+                    "pack_qty": "",
+                    "measure_type": "weight",
+                    "is_store_brand": "true",
+                    "is_fee": "false",
+                    "line_total": "7.99",
+                }
+            ]
+
+            write_csv_rows(
+                observed_path, observed_rows, build_observed_products.OUTPUT_FIELDS
+            )
+            write_csv_rows(items_path, item_rows, list(item_rows[0].keys()))
+
+            build_review_queue.main.callback(
+                observed_csv=str(observed_path),
+                items_enriched_csv=str(items_path),
+                output_csv=str(output_path),
+            )
+
+            self.assertTrue(output_path.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()