Record t1.8.5 task evidence

agents.md

@@ -7,6 +7,7 @@
 ## tech stack
 - python; pandas or polars
 - file storage: json and csv, no sqlite or databases
+- assume local virtual env is available and accessible
 - do not add new dependencies unless explicitly approved; if unavoidable, document justification in the active task notes
 
 ## workflow

build_canonical_layer.py

@@ -58,7 +58,11 @@ def normalized_quantity(row):
 
 
 def auto_link_rule(observed_row):
-    if observed_row.get("is_fee") == "true":
+    if (
+        observed_row.get("is_fee") == "true"
+        or observed_row.get("is_discount_line") == "true"
+        or observed_row.get("is_coupon_line") == "true"
+    ):
         return "", "", ""
 
     if observed_row.get("representative_upc"):

build_observed_products.py

@@ -17,6 +17,7 @@ OUTPUT_FIELDS = [
     "observed_product_id",
     "retailer",
     "observed_key",
+    "representative_retailer_item_id",
     "representative_upc",
     "representative_item_name",
     "representative_name_norm",
@@ -29,6 +30,8 @@ OUTPUT_FIELDS = [
     "representative_image_url",
     "is_store_brand",
     "is_fee",
+    "is_discount_line",
+    "is_coupon_line",
     "first_seen_date",
     "last_seen_date",
     "times_seen",
@@ -38,6 +41,7 @@ OUTPUT_FIELDS = [
     "normalized_name_examples",
     "example_prices",
     "distinct_item_names_count",
+    "distinct_retailer_item_ids_count",
     "distinct_upcs_count",
 ]
 
@@ -52,6 +56,17 @@ def build_observed_key(row):
             ]
         )
 
+    if row.get("retailer_item_id"):
+        return "|".join(
+            [
+                row["retailer"],
+                f"retailer_item_id={row['retailer_item_id']}",
+                f"name={row['item_name_norm']}",
+                f"discount={row.get('is_discount_line', 'false')}",
+                f"coupon={row.get('is_coupon_line', 'false')}",
+            ]
+        )
+
     return "|".join(
         [
             row["retailer"],
@@ -82,6 +97,9 @@ def build_observed_products(rows):
                 "observed_product_id": stable_id("gobs", observed_key),
                 "retailer": ordered[0]["retailer"],
                 "observed_key": observed_key,
+                "representative_retailer_item_id": representative_value(
+                    ordered, "retailer_item_id"
+                ),
                 "representative_upc": representative_value(ordered, "upc"),
                 "representative_item_name": representative_value(ordered, "item_name"),
                 "representative_name_norm": representative_value(
@@ -98,6 +116,10 @@ def build_observed_products(rows):
                 "representative_image_url": first_nonblank(ordered, "image_url"),
                 "is_store_brand": representative_value(ordered, "is_store_brand"),
                 "is_fee": representative_value(ordered, "is_fee"),
+                "is_discount_line": representative_value(
+                    ordered, "is_discount_line"
+                ),
+                "is_coupon_line": representative_value(ordered, "is_coupon_line"),
                 "first_seen_date": ordered[0]["order_date"],
                 "last_seen_date": ordered[-1]["order_date"],
                 "times_seen": str(len(ordered)),
@@ -115,6 +137,9 @@ def build_observed_products(rows):
                 "distinct_item_names_count": str(
                     len(distinct_values(ordered, "item_name"))
                 ),
+                "distinct_retailer_item_ids_count": str(
+                    len(distinct_values(ordered, "retailer_item_id"))
+                ),
                 "distinct_upcs_count": str(len(distinct_values(ordered, "upc"))),
             }
         )

build_review_queue.py

@@ -37,7 +37,11 @@ def existing_review_state(path):
 
 def review_reasons(observed_row):
     reasons = []
-    if observed_row["is_fee"] == "true":
+    if (
+        observed_row["is_fee"] == "true"
+        or observed_row.get("is_discount_line") == "true"
+        or observed_row.get("is_coupon_line") == "true"
+    ):
         return reasons
     if observed_row["distinct_upcs_count"] not in {"", "0", "1"}:
         reasons.append(("multiple_upcs", "high"))
@@ -119,6 +123,7 @@ def attach_observed_ids(item_rows, observed_rows):
         ) if row.get("upc") else "|".join(
             [
                 row["retailer"],
+                f"retailer_item_id={row.get('retailer_item_id', '')}",
                 f"name={row['item_name_norm']}",
                 f"size={row['size_value']}",
                 f"unit={row['size_unit']}",
@@ -126,6 +131,8 @@ def attach_observed_ids(item_rows, observed_rows):
                 f"measure={row['measure_type']}",
                 f"store_brand={row['is_store_brand']}",
                 f"fee={row['is_fee']}",
+                f"discount={row.get('is_discount_line', 'false')}",
+                f"coupon={row.get('is_coupon_line', 'false')}",
             ]
         )
         enriched = dict(row)

enrich_costco.py

@@ -0,0 +1,271 @@
+import csv
+import json
+import re
+from pathlib import Path
+
+import click
+
+from enrich_giant import (
+    OUTPUT_FIELDS,
+    format_decimal,
+    normalize_number,
+    normalize_unit,
+    normalize_whitespace,
+    singularize_tokens,
+    to_decimal,
+)
+
+
+PARSER_VERSION = "costco-enrich-v1"
+RETAILER = "costco"
+DEFAULT_INPUT_DIR = Path("costco_output/raw")
+DEFAULT_OUTPUT_CSV = Path("costco_output/items_enriched.csv")
+
+CODE_TOKEN_RE = re.compile(
+    r"\b(?:SL\d+|T\d+H\d+|P\d+(?:/\d+)?|W\d+T\d+H\d+|FY\d+|CSPC#|C\d+T\d+H\d+|EC\d+T\d+H\d+|\d+X\d+)\b"
+)
+PACK_FRACTION_RE = re.compile(r"(?<![A-Z0-9])(\d+)\s*/\s*(\d+(?:\.\d+)?)\s*(OZ|LB|LBS|CT)\b")
+HASH_SIZE_RE = re.compile(r"(?<![A-Z0-9])(\d+(?:\.\d+)?)#\b")
+PACK_DASH_RE = re.compile(r"(?<![A-Z0-9])(\d+)\s*-\s*PACK\b")
+PACK_WORD_RE = re.compile(r"(?<![A-Z0-9])(\d+)\s*PACK\b")
+SIZE_RE = re.compile(r"(?<![A-Z0-9])(\d+(?:\.\d+)?)\s*(OZ|LB|LBS|CT|KG|G)\b")
+
+
+def clean_costco_name(name):
+    cleaned = normalize_whitespace(name).upper().replace('"', "")
+    cleaned = CODE_TOKEN_RE.sub(" ", cleaned)
+    cleaned = re.sub(r"\s*/\s*\d+(?:\.\d+)?\s*(KG|G)\b", " ", cleaned)
+    cleaned = normalize_whitespace(cleaned)
+    return cleaned
+
+
+def combine_description(item):
+    return normalize_whitespace(
+        " ".join(
+            str(part).strip()
+            for part in [item.get("itemDescription01"), item.get("itemDescription02")]
+            if part
+        )
+    )
+
+
+def parse_costco_size_and_pack(cleaned_name):
+    pack_qty = ""
+    size_value = ""
+    size_unit = ""
+
+    match = PACK_FRACTION_RE.search(cleaned_name)
+    if match:
+        pack_qty = normalize_number(match.group(1))
+        size_value = normalize_number(match.group(2))
+        size_unit = normalize_unit(match.group(3))
+        return size_value, size_unit, pack_qty
+
+    match = HASH_SIZE_RE.search(cleaned_name)
+    if match:
+        size_value = normalize_number(match.group(1))
+        size_unit = "lb"
+
+    match = PACK_DASH_RE.search(cleaned_name) or PACK_WORD_RE.search(cleaned_name)
+    if match:
+        pack_qty = normalize_number(match.group(1))
+
+    matches = list(SIZE_RE.finditer(cleaned_name))
+    if matches:
+        last = matches[-1]
+        unit = last.group(2)
+        size_value = normalize_number(last.group(1))
+        size_unit = "count" if unit == "CT" else normalize_unit(unit)
+
+    return size_value, size_unit, pack_qty
+
+
+def normalize_costco_name(cleaned_name):
+    brand = ""
+    base = cleaned_name
+    if base.startswith("KS "):
+        brand = "KS"
+        base = normalize_whitespace(base[3:])
+
+    size_value, size_unit, pack_qty = parse_costco_size_and_pack(base)
+    if size_value and size_unit:
+        if pack_qty:
+            base = PACK_FRACTION_RE.sub(" ", base)
+        else:
+            base = SIZE_RE.sub(" ", base)
+    base = HASH_SIZE_RE.sub(" ", base)
+    base = PACK_DASH_RE.sub(" ", base)
+    base = PACK_WORD_RE.sub(" ", base)
+    base = normalize_whitespace(base)
+    tokens = []
+    for token in base.split():
+        if token in {"ORG"}:
+            continue
+        if token in {"PEANUT", "BUTTER"} and "JIF" in base:
+            continue
+        tokens.append(token)
+    base = singularize_tokens(" ".join(tokens))
+    return normalize_whitespace(base), brand, size_value, size_unit, pack_qty
+
+
+def guess_measure_type(size_unit, pack_qty, is_discount_line):
+    if is_discount_line:
+        return "each"
+    if size_unit in {"lb", "oz", "g", "kg"}:
+        return "weight"
+    if size_unit in {"ml", "l", "qt", "pt", "gal", "fl_oz"}:
+        return "volume"
+    if size_unit == "count" or pack_qty:
+        return "count"
+    return "each"
+
+
+def derive_costco_prices(item, measure_type, size_value, size_unit, pack_qty):
+    line_total = to_decimal(item.get("amount"))
+    qty = to_decimal(item.get("unit"))
+    parsed_size = to_decimal(size_value)
+    parsed_pack = to_decimal(pack_qty) or 1
+
+    price_per_each = ""
+    price_per_lb = ""
+    price_per_oz = ""
+    if line_total is None:
+        return price_per_each, price_per_lb, price_per_oz
+
+    if measure_type in {"each", "count"} and qty not in (None, 0):
+        price_per_each = format_decimal(line_total / qty)
+
+    if parsed_size not in (None, 0):
+        total_units = parsed_size * parsed_pack * (qty or 1)
+        if size_unit == "lb":
+            per_lb = line_total / total_units
+            price_per_lb = format_decimal(per_lb)
+            price_per_oz = format_decimal(per_lb / 16)
+        elif size_unit == "oz":
+            per_oz = line_total / total_units
+            price_per_oz = format_decimal(per_oz)
+            price_per_lb = format_decimal(per_oz * 16)
+
+    return price_per_each, price_per_lb, price_per_oz
+
+
+def is_discount_item(item):
+    amount = to_decimal(item.get("amount")) or 0
+    unit = to_decimal(item.get("unit")) or 0
+    description = combine_description(item)
+    return amount < 0 or unit < 0 or description.startswith("/")
+
+
+def parse_costco_item(order_id, order_date, raw_path, line_no, item):
+    raw_name = combine_description(item)
+    cleaned_name = clean_costco_name(raw_name)
+    item_name_norm, brand_guess, size_value, size_unit, pack_qty = normalize_costco_name(
+        cleaned_name
+    )
+    is_discount_line = is_discount_item(item)
+    is_coupon_line = "true" if raw_name.startswith("/") else "false"
+    measure_type = guess_measure_type(size_unit, pack_qty, is_discount_line)
+    price_per_each, price_per_lb, price_per_oz = derive_costco_prices(
+        item, measure_type, size_value, size_unit, pack_qty
+    )
+
+    return {
+        "retailer": RETAILER,
+        "order_id": str(order_id),
+        "line_no": str(line_no),
+        "observed_item_key": f"{RETAILER}:{order_id}:{line_no}",
+        "order_date": normalize_whitespace(order_date),
+        "retailer_item_id": str(item.get("itemNumber", "")),
+        "pod_id": "",
+        "item_name": raw_name,
+        "upc": "",
+        "category_id": str(item.get("itemDepartmentNumber", "")),
+        "category": str(item.get("transDepartmentNumber", "")),
+        "qty": str(item.get("unit", "")),
+        "unit": str(item.get("itemIdentifier", "")),
+        "unit_price": str(item.get("itemUnitPriceAmount", "")),
+        "line_total": str(item.get("amount", "")),
+        "picked_weight": "",
+        "mvp_savings": "",
+        "reward_savings": "",
+        "coupon_savings": str(item.get("amount", "")) if is_discount_line else "",
+        "coupon_price": "",
+        "image_url": "",
+        "raw_order_path": raw_path.as_posix(),
+        "item_name_norm": item_name_norm,
+        "brand_guess": brand_guess,
+        "variant": "",
+        "size_value": size_value,
+        "size_unit": size_unit,
+        "pack_qty": pack_qty,
+        "measure_type": measure_type,
+        "is_store_brand": "true" if brand_guess else "false",
+        "is_fee": "false",
+        "is_discount_line": "true" if is_discount_line else "false",
+        "is_coupon_line": is_coupon_line,
+        "price_per_each": price_per_each,
+        "price_per_lb": price_per_lb,
+        "price_per_oz": price_per_oz,
+        "parse_version": PARSER_VERSION,
+        "parse_notes": "",
+    }
+
+
+def iter_costco_rows(raw_dir):
+    for path in discover_json_files(raw_dir):
+        if path.name == "summary.json":
+            continue
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        receipts = payload.get("data", {}).get("receiptsWithCounts", {}).get("receipts", [])
+        for receipt in receipts:
+            order_id = receipt["transactionBarcode"]
+            order_date = receipt.get("transactionDate", "")
+            for line_no, item in enumerate(receipt.get("itemArray", []), start=1):
+                yield parse_costco_item(order_id, order_date, path, line_no, item)
+
+
+def discover_json_files(raw_dir):
+    raw_dir = Path(raw_dir)
+    candidates = sorted(raw_dir.glob("*.json"))
+    if candidates:
+        return candidates
+    if raw_dir.name == "raw" and raw_dir.parent.exists():
+        return sorted(raw_dir.parent.glob("*.json"))
+    return []
+
+
+def build_items_enriched(raw_dir):
+    rows = list(iter_costco_rows(raw_dir))
+    rows.sort(key=lambda row: (row["order_date"], row["order_id"], int(row["line_no"])))
+    return rows
+
+
+def write_csv(path, rows):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=OUTPUT_FIELDS)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+@click.command()
+@click.option(
+    "--input-dir",
+    default=str(DEFAULT_INPUT_DIR),
+    show_default=True,
+    help="Directory containing Costco raw order json files.",
+)
+@click.option(
+    "--output-csv",
+    default=str(DEFAULT_OUTPUT_CSV),
+    show_default=True,
+    help="CSV path for enriched Costco item rows.",
+)
+def main(input_dir, output_csv):
+    rows = build_items_enriched(Path(input_dir))
+    write_csv(Path(output_csv), rows)
+    click.echo(f"wrote {len(rows)} rows to {output_csv}")
+
+
+if __name__ == "__main__":
+    main()

enrich_giant.py

@@ -18,6 +18,7 @@ OUTPUT_FIELDS = [
     "line_no",
     "observed_item_key",
     "order_date",
+    "retailer_item_id",
     "pod_id",
     "item_name",
     "upc",
@@ -43,6 +44,8 @@ OUTPUT_FIELDS = [
     "measure_type",
     "is_store_brand",
     "is_fee",
+    "is_discount_line",
+    "is_coupon_line",
     "price_per_each",
     "price_per_lb",
     "price_per_oz",
@@ -55,6 +58,8 @@ STORE_BRAND_PREFIXES = {
     "NP": "NP",
 }
 
+DROP_TOKENS = {"FRESH"}
+
 ABBREVIATIONS = {
     "APPLE": "APPLE",
     "APPLES": "APPLES",
@@ -234,9 +239,30 @@ def normalize_item_name(cleaned_name):
         base = normalize_whitespace(base[len(prefix):])
 
     base = strip_measure_tokens(base)
-    expanded_tokens = [expand_token(token) for token in base.split()]
+    expanded_tokens = []
+    for token in base.split():
+        expanded = expand_token(token)
+        if expanded in DROP_TOKENS:
+            continue
+        expanded_tokens.append(expanded)
     expanded = " ".join(token for token in expanded_tokens if token)
-    return normalize_whitespace(expanded)
+    return singularize_tokens(normalize_whitespace(expanded))
+
+
+def singularize_tokens(text):
+    singular_map = {
+        "APPLES": "APPLE",
+        "BANANAS": "BANANA",
+        "BERRIES": "BERRY",
+        "EGGS": "EGG",
+        "LEMONS": "LEMON",
+        "LIMES": "LIME",
+        "MANDARINS": "MANDARIN",
+        "PEPPERS": "PEPPER",
+        "STRAWBERRIES": "STRAWBERRY",
+    }
+    tokens = [singular_map.get(token, token) for token in text.split()]
+    return normalize_whitespace(" ".join(tokens))
 
 
 def guess_measure_type(item, size_unit, pack_qty):
@@ -330,6 +356,7 @@ def parse_item(order_id, order_date, raw_path, line_no, item):
         "line_no": str(line_no),
         "observed_item_key": f"{RETAILER}:{order_id}:{line_no}",
         "order_date": normalize_whitespace(order_date),
+        "retailer_item_id": stringify(item.get("podId")),
         "pod_id": stringify(item.get("podId")),
         "item_name": stringify(item.get("itemName")),
         "upc": stringify(item.get("primUpcCd")),
@@ -355,6 +382,8 @@ def parse_item(order_id, order_date, raw_path, line_no, item):
         "measure_type": measure_type,
         "is_store_brand": "true" if bool(prefix) else "false",
         "is_fee": "true" if is_fee else "false",
+        "is_discount_line": "false",
+        "is_coupon_line": "false",
         "price_per_each": price_per_each,
         "price_per_lb": price_per_lb,
         "price_per_oz": price_per_oz,

pm/data-model.org

@@ -129,6 +129,7 @@ One row per retailer line item.
 | `order_id`       | retailer order id                       |
 | `line_no`        | stable line number within order export  |
 | `order_date`     | copied from order when available        |
+| `retailer_item_id` | retailer-native item id when available |
 | `pod_id`         | retailer pod/item id                    |
 | `item_name`      | raw retailer item name                  |
 | `upc`            | retailer UPC or PLU value               |
@@ -145,6 +146,8 @@ One row per retailer line item.
 | `coupon_price`   | retailer coupon price field             |
 | `image_url`      | raw retailer image url when present     |
 | `raw_order_path` | relative path to source order payload   |
+| `is_discount_line` | retailer adjustment or discount-line flag |
+| `is_coupon_line` | coupon-like line flag when distinguishable |
 
 Primary key:
 
@@ -161,6 +164,7 @@ fields from `items_raw.csv` and add parsed fields.
 | `order_id`          | retailer order id                                           |
 | `line_no`           | line number within order                                    |
 | `observed_item_key` | stable row key, typically `<retailer>:<order_id>:<line_no>` |
+| `retailer_item_id`  | retailer-native item id                                     |
 | `item_name`         | raw retailer item name                                      |
 | `item_name_norm`    | normalized item name                                        |
 | `brand_guess`       | parsed brand guess                                          |
@@ -171,6 +175,8 @@ fields from `items_raw.csv` and add parsed fields.
 | `measure_type`      | `each`, `weight`, `volume`, `count`, or blank               |
 | `is_store_brand`    | store-brand guess                                           |
 | `is_fee`            | fee or non-product flag                                     |
+| `is_discount_line`  | discount or adjustment-line flag                            |
+| `is_coupon_line`    | coupon-like line flag                                       |
 | `price_per_each`    | derived per-each price when supported                       |
 | `price_per_lb`      | derived per-pound price when supported                      |
 | `price_per_oz`      | derived per-ounce price when supported                      |
@@ -191,6 +197,7 @@ One row per distinct retailer-facing observed product.
 | `observed_product_id`         | stable observed product id                                     |
 | `retailer`                    | retailer slug                                                  |
 | `observed_key`                | deterministic grouping key used to create the observed product |
+| `representative_retailer_item_id` | best representative retailer-native item id               |
 | `representative_upc`          | best representative UPC/PLU                                    |
 | `representative_item_name`    | representative raw retailer name                               |
 | `representative_name_norm`    | representative normalized name                                 |
@@ -203,11 +210,14 @@ One row per distinct retailer-facing observed product.
 | `representative_image_url`    | representative image url                                       |
 | `is_store_brand`              | representative store-brand flag                                |
 | `is_fee`                      | representative fee flag                                        |
+| `is_discount_line`            | representative discount-line flag                              |
+| `is_coupon_line`              | representative coupon-line flag                                |
 | `first_seen_date`             | first order date seen                                          |
 | `last_seen_date`              | last order date seen                                           |
 | `times_seen`                  | number of enriched item rows grouped here                      |
 | `example_order_id`            | one example retailer order id                                  |
 | `example_item_name`           | one example raw item name                                      |
+| `distinct_retailer_item_ids_count` | count of distinct retailer-native item ids               |
 
 Primary key:
 
@@ -297,4 +307,3 @@ Current scraper outputs map to the new layout as follows:
 Current Giant raw order payloads already expose fields needed for future
 enrichment, including `image`, `itemName`, `primUpcCd`, `lbEachCd`,
 `unitPrice`, `groceryAmount`, and `totalPickedWeight`.
-

pm/tasks.org

@@ -143,7 +143,7 @@
 - tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python build_canonical_layer.py`; verified auto-linked `giant_output/products_canonical.csv` and `giant_output/product_links.csv`
 - date: 2026-03-16
 
-* [ ] t1.8: support costco raw ingest path (2-5 commits)
+* [X] t1.8: support costco raw ingest path (2-5 commits)
 
 ** acceptance criteria
 - add a costco-specific raw ingest/export path
@@ -158,11 +158,11 @@
 - bearer/auth values should come from local env, not source
 
 ** evidence
-- commit:
-- tests:
-- date:
+- commit: `da00288` on branch `cx`
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python scrape_costco.py --help`; verified `costco_output/raw/*.json`, `costco_output/orders.csv`, and `costco_output/items.csv` from the local sample payload
+- date: 2026-03-16
 
-* [ ] t1.8.1: support costco parser/enricher path (2-4 commits)
+* [X] t1.8.1: support costco parser/enricher path (2-4 commits)
 
 ** acceptance criteria
 - add a costco-specific enrich step producing `costco_output/items_enriched.csv`
@@ -179,10 +179,10 @@
 - expect weaker identifiers than Giant
 
 ** evidence
-- commit:
-- tests:
-- date:
-* [ ] t1.8.2: validate cross-retailer observed/canonical flow (1-3 commits)
+- commit: `da00288` on branch `cx`
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python enrich_costco.py`; verified `costco_output/items_enriched.csv`
+- date: 2026-03-16
+* [X] t1.8.2: validate cross-retailer observed/canonical flow (1-3 commits)
 
 ** acceptance criteria
 - feed Giant and Costco enriched rows through the same observed/canonical pipeline
@@ -197,10 +197,10 @@
 - apples, eggs, bananas, or flour are better than weird prepared foods
 
 ** evidence
-- commit:
-- tests:
-- date:
-* [ ] t1.8.3: extend shared schema for retailer-native ids and adjustment lines (1-2 commits)
+- commit: `da00288` on branch `cx`
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python validate_cross_retailer_flow.py`; proof example: Giant `FRESH BANANA` and Costco `BANANAS 3 LB / 1.36 KG` share one canonical in `combined_output/proof_examples.csv`
+- date: 2026-03-16
+* [X] t1.8.3: extend shared schema for retailer-native ids and adjustment lines (1-2 commits)
 
 ** acceptance criteria
 - add shared fields needed for non-upc retailers, including:
@@ -215,9 +215,45 @@
 - do this once instead of sprinkling exceptions everywhere
 
 ** evidence
-- commit:
-- tests:
-- date:
+- commit: `9497565` on branch `cx`
+- tests: `./venv/bin/python -m unittest discover -s tests`; verified shared enriched fields in `giant_output/items_enriched.csv` and `costco_output/items_enriched.csv`
+- date: 2026-03-16
+* [X] t1.8.4: verify and correct costco receipt enumeration (1–2 commits)
+
+** acceptance criteria
+- confirm graphql summary query returns all expected receipts
+- compare `inWarehouse` count vs number of `receipts` returned
+- widen or parameterize date window if necessary; website shows receipts in 3-month windows
+- persist request metadata (`startDate`, `endDate`, `documentType`, `documentSubType`)
+- emit warning when receipt counts mismatch
+
+** notes
+- goal is to confirm we are enumerating all receipts before parsing
+- do not expand schema or parser logic in this task
+- keep changes limited to summary query handling and diagnostics
+
+** evidence
+- commit: `ac82fa6` on branch `cx`
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python scrape_costco.py --help`; reviewed the sample Costco summary request in `pm/scrape-giant.org` against `costco_output/raw/summary.json` and added 3-month window chunking plus mismatch diagnostics
+- date: 2026-03-16
+* [X] t1.8.5: refactor costco scraper auth and UX with giant scraper
+
+** acceptance criteria
+- remove manual auth env vars
+- load costco cookies from firefox session
+- require only logged-in browser
+- replace start/end date flags with --months-back
+- maintain same raw output structure
+- ensure summary_lookup keys are collision-safe by using a composite key (transactionBarcode + transactionDateTime) instead of transactionBarcode alone 
+
+** notes
+- align Costco acquisition ergonomics with the Giant scraper
+- keep downstream Costco parsing and shared schemas unchanged
+
+** evidence
+- commit: `c0054dc` on branch `cx`
+- tests: `./venv/bin/python -m unittest discover -s tests`; `./venv/bin/python scrape_costco.py --help`; verified Costco summary/detail flattening now uses composite receipt keys in unit tests
+- date: 2026-03-16
 * [ ] t1.9: compute normalized comparison metrics (2-4 commits)
 
 ** acceptance criteria

scrape_costco.py

@@ -0,0 +1,619 @@
+import csv
+import json
+import time
+from calendar import monthrange
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import click
+import browser_cookie3
+from curl_cffi import requests
+
+BASE_URL = "https://ecom-api.costco.com/ebusiness/order/v1/orders/graphql"
+RETAILER = "costco"
+
+SUMMARY_QUERY = """
+query receiptsWithCounts($startDate: String!, $endDate: String!, $documentType: String!, $documentSubType: String!) {
+  receiptsWithCounts(startDate: $startDate, endDate: $endDate, documentType: $documentType, documentSubType: $documentSubType) {
+    inWarehouse
+    gasStation
+    carWash
+    gasAndCarWash
+    receipts {
+      warehouseName
+      receiptType
+      documentType
+      transactionDateTime
+      transactionBarcode
+      warehouseName
+      transactionType
+      total
+      totalItemCount
+      itemArray {
+        itemNumber
+      }
+      tenderArray {
+        tenderTypeCode
+        tenderDescription
+        amountTender
+      }
+      couponArray {
+        upcnumberCoupon
+      }
+    }
+  }
+}
+""".strip()
+
+DETAIL_QUERY = """
+query receiptsWithCounts($barcode: String!, $documentType: String!) {
+  receiptsWithCounts(barcode: $barcode, documentType: $documentType) {
+    receipts {
+      warehouseName
+      receiptType
+      documentType
+      transactionDateTime
+      transactionDate
+      companyNumber
+      warehouseNumber
+      operatorNumber
+      warehouseShortName
+      registerNumber
+      transactionNumber
+      transactionType
+      transactionBarcode
+      total
+      warehouseAddress1
+      warehouseAddress2
+      warehouseCity
+      warehouseState
+      warehouseCountry
+      warehousePostalCode
+      totalItemCount
+      subTotal
+      taxes
+      total
+      invoiceNumber
+      sequenceNumber
+      itemArray {
+        itemNumber
+        itemDescription01
+        frenchItemDescription1
+        itemDescription02
+        frenchItemDescription2
+        itemIdentifier
+        itemDepartmentNumber
+        unit
+        amount
+        taxFlag
+        merchantID
+        entryMethod
+        transDepartmentNumber
+        fuelUnitQuantity
+        fuelGradeCode
+        itemUnitPriceAmount
+        fuelUomCode
+        fuelUomDescription
+        fuelUomDescriptionFr
+        fuelGradeDescription
+        fuelGradeDescriptionFr
+      }
+      tenderArray {
+        tenderTypeCode
+        tenderSubTypeCode
+        tenderDescription
+        amountTender
+        displayAccountNumber
+        sequenceNumber
+        approvalNumber
+        responseCode
+        tenderTypeName
+        transactionID
+        merchantID
+        entryMethod
+        tenderAcctTxnNumber
+        tenderAuthorizationCode
+        tenderTypeNameFr
+        tenderEntryMethodDescription
+        walletType
+        walletId
+        storedValueBucket
+      }
+      subTaxes {
+        tax1
+        tax2
+        tax3
+        tax4
+        aTaxPercent
+        aTaxLegend
+        aTaxAmount
+        aTaxPrintCode
+        aTaxPrintCodeFR
+        aTaxIdentifierCode
+        bTaxPercent
+        bTaxLegend
+        bTaxAmount
+        bTaxPrintCode
+        bTaxPrintCodeFR
+        bTaxIdentifierCode
+        cTaxPercent
+        cTaxLegend
+        cTaxAmount
+        cTaxIdentifierCode
+        dTaxPercent
+        dTaxLegend
+        dTaxAmount
+        dTaxPrintCode
+        dTaxPrintCodeFR
+        dTaxIdentifierCode
+        uTaxLegend
+        uTaxAmount
+        uTaxableAmount
+      }
+      instantSavings
+      membershipNumber
+    }
+  }
+}
+""".strip()
+
+ORDER_FIELDS = [
+    "retailer",
+    "order_id",
+    "order_date",
+    "delivery_date",
+    "service_type",
+    "order_total",
+    "payment_method",
+    "total_item_count",
+    "total_savings",
+    "your_savings_total",
+    "coupons_discounts_total",
+    "store_name",
+    "store_number",
+    "store_address1",
+    "store_city",
+    "store_state",
+    "store_zipcode",
+    "refund_order",
+    "ebt_order",
+    "raw_history_path",
+    "raw_order_path",
+]
+
+ITEM_FIELDS = [
+    "retailer",
+    "order_id",
+    "line_no",
+    "order_date",
+    "retailer_item_id",
+    "pod_id",
+    "item_name",
+    "upc",
+    "category_id",
+    "category",
+    "qty",
+    "unit",
+    "unit_price",
+    "line_total",
+    "picked_weight",
+    "mvp_savings",
+    "reward_savings",
+    "coupon_savings",
+    "coupon_price",
+    "image_url",
+    "raw_order_path",
+    "is_discount_line",
+    "is_coupon_line",
+]
+
+
+def build_headers():
+    return {
+        "accept": "*/*",
+        "content-type": "application/json-patch+json",
+        "costco.service": "restOrders",
+        "costco.env": "ecom",
+        "origin": "https://www.costco.com",
+        "referer": "https://www.costco.com/",
+        "user-agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) "
+            "Gecko/20100101 Firefox/148.0"
+        ),
+    }
+
+
+def build_session():
+    session = requests.Session()
+    session.cookies.update(browser_cookie3.firefox(domain_name="costco.com"))
+    session.headers.update(build_headers())
+    return session
+
+
+def graphql_post(session, query, variables):
+    last_response = None
+
+    for attempt in range(3):
+        try:
+            response = session.post(
+                BASE_URL,
+                json={"query": query, "variables": variables},
+                impersonate="firefox",
+                timeout=30,
+            )
+            last_response = response
+            if response.status_code == 200:
+                return response.json()
+            click.echo(f"retry {attempt + 1}/3 status={response.status_code}")
+        except Exception as exc:  # pragma: no cover - network error path
+            click.echo(f"retry {attempt + 1}/3 error={exc}")
+        time.sleep(3)
+
+    if last_response is not None:
+        last_response.raise_for_status()
+
+    raise RuntimeError("failed to fetch Costco GraphQL payload")
+
+
+def summary_receipts(payload):
+    return payload.get("data", {}).get("receiptsWithCounts", {}).get("receipts", [])
+
+
+def detail_receipts(payload):
+    return payload.get("data", {}).get("receiptsWithCounts", {}).get("receipts", [])
+
+
+def summary_counts(payload):
+    counts = payload.get("data", {}).get("receiptsWithCounts", {})
+    return {
+        "inWarehouse": counts.get("inWarehouse", 0) or 0,
+        "gasStation": counts.get("gasStation", 0) or 0,
+        "carWash": counts.get("carWash", 0) or 0,
+        "gasAndCarWash": counts.get("gasAndCarWash", 0) or 0,
+    }
+
+
+def parse_cli_date(value):
+    return datetime.strptime(value, "%m/%d/%Y").date()
+
+
+def format_cli_date(value):
+    return f"{value.month}/{value.day:02d}/{value.year}"
+
+
+def subtract_months(value, months):
+    year = value.year
+    month = value.month - months
+    while month <= 0:
+        month += 12
+        year -= 1
+    day = min(value.day, monthrange(year, month)[1])
+    return value.replace(year=year, month=month, day=day)
+
+
+def resolve_date_range(months_back, today=None):
+    if months_back < 1:
+        raise click.ClickException("months-back must be at least 1")
+
+    end = today or datetime.now().date()
+    start = subtract_months(end, months_back)
+    return format_cli_date(start), format_cli_date(end)
+
+
+def build_date_windows(start_date, end_date, window_days):
+    start = parse_cli_date(start_date)
+    end = parse_cli_date(end_date)
+    if end < start:
+        raise click.ClickException("end-date must be on or after start-date")
+    if window_days < 1:
+        raise click.ClickException("window-days must be at least 1")
+
+    windows = []
+    current = start
+    while current <= end:
+        window_end = min(current + timedelta(days=window_days - 1), end)
+        windows.append(
+            {
+                "startDate": format_cli_date(current),
+                "endDate": format_cli_date(window_end),
+            }
+        )
+        current = window_end + timedelta(days=1)
+    return windows
+
+
+def unique_receipts(receipts):
+    by_barcode = {}
+    for receipt in receipts:
+        key = receipt_key(receipt)
+        if key:
+            by_barcode[key] = receipt
+    return list(by_barcode.values())
+
+
+def receipt_key(receipt):
+    barcode = receipt.get("transactionBarcode", "")
+    transaction_date_time = receipt.get("transactionDateTime", "")
+    if not barcode:
+        return ""
+    return f"{barcode}::{transaction_date_time}"
+
+
+def fetch_summary_windows(
+    session,
+    start_date,
+    end_date,
+    document_type,
+    document_sub_type,
+    window_days,
+):
+    requests_metadata = []
+    combined_receipts = []
+
+    for window in build_date_windows(start_date, end_date, window_days):
+        variables = {
+            "startDate": window["startDate"],
+            "endDate": window["endDate"],
+            "text": "custom",
+            "documentType": document_type,
+            "documentSubType": document_sub_type,
+        }
+        payload = graphql_post(session, SUMMARY_QUERY, variables)
+        receipts = summary_receipts(payload)
+        counts = summary_counts(payload)
+        warehouse_count = sum(
+            1 for receipt in receipts if receipt.get("receiptType") == "In-Warehouse"
+        )
+        mismatch = counts["inWarehouse"] != warehouse_count
+        requests_metadata.append(
+            {
+                **variables,
+                "returnedReceipts": len(receipts),
+                "returnedInWarehouseReceipts": warehouse_count,
+                "inWarehouse": counts["inWarehouse"],
+                "gasStation": counts["gasStation"],
+                "carWash": counts["carWash"],
+                "gasAndCarWash": counts["gasAndCarWash"],
+                "countMismatch": mismatch,
+            }
+        )
+        if mismatch:
+            click.echo(
+                (
+                    "warning: summary count mismatch for "
+                    f"{window['startDate']} to {window['endDate']}: "
+                    f"inWarehouse={counts['inWarehouse']} "
+                    f"returnedInWarehouseReceipts={warehouse_count}"
+                ),
+                err=True,
+            )
+        combined_receipts.extend(receipts)
+
+    unique = unique_receipts(combined_receipts)
+    aggregate_payload = {
+        "data": {
+            "receiptsWithCounts": {
+                "inWarehouse": sum(row["inWarehouse"] for row in requests_metadata),
+                "gasStation": sum(row["gasStation"] for row in requests_metadata),
+                "carWash": sum(row["carWash"] for row in requests_metadata),
+                "gasAndCarWash": sum(row["gasAndCarWash"] for row in requests_metadata),
+                "receipts": unique,
+            }
+        }
+    }
+    return aggregate_payload, requests_metadata
+
+
+def flatten_costco_data(summary_payload, detail_payloads, raw_dir):
+    summary_lookup = {
+        receipt_key(receipt): receipt
+        for receipt in summary_receipts(summary_payload)
+        if receipt_key(receipt)
+    }
+    orders = []
+    items = []
+
+    for detail_payload in detail_payloads:
+        for receipt in detail_receipts(detail_payload):
+            order_id = receipt["transactionBarcode"]
+            receipt_id = receipt_key(receipt)
+            summary_row = summary_lookup.get(receipt_id, {})
+            coupon_numbers = {
+                row.get("upcnumberCoupon", "")
+                for row in summary_row.get("couponArray", []) or []
+                if row.get("upcnumberCoupon")
+            }
+            raw_order_path = raw_dir / f"{receipt_id or order_id}.json"
+
+            orders.append(
+                {
+                    "retailer": RETAILER,
+                    "order_id": order_id,
+                    "order_date": receipt.get("transactionDate", ""),
+                    "delivery_date": receipt.get("transactionDate", ""),
+                    "service_type": receipt.get("receiptType", ""),
+                    "order_total": stringify(receipt.get("total")),
+                    "payment_method": compact_join(
+                        summary_row.get("tenderArray", []) or [], "tenderDescription"
+                    ),
+                    "total_item_count": stringify(receipt.get("totalItemCount")),
+                    "total_savings": stringify(receipt.get("instantSavings")),
+                    "your_savings_total": stringify(receipt.get("instantSavings")),
+                    "coupons_discounts_total": stringify(receipt.get("instantSavings")),
+                    "store_name": receipt.get("warehouseName", ""),
+                    "store_number": stringify(receipt.get("warehouseNumber")),
+                    "store_address1": receipt.get("warehouseAddress1", ""),
+                    "store_city": receipt.get("warehouseCity", ""),
+                    "store_state": receipt.get("warehouseState", ""),
+                    "store_zipcode": receipt.get("warehousePostalCode", ""),
+                    "refund_order": "false",
+                    "ebt_order": "false",
+                    "raw_history_path": (raw_dir / "summary.json").as_posix(),
+                    "raw_order_path": raw_order_path.as_posix(),
+                }
+            )
+
+            for line_no, item in enumerate(receipt.get("itemArray", []), start=1):
+                item_number = stringify(item.get("itemNumber"))
+                description = join_descriptions(
+                    item.get("itemDescription01"), item.get("itemDescription02")
+                )
+                is_discount = is_discount_line(item)
+                is_coupon = is_discount and (
+                    item_number in coupon_numbers
+                    or description.startswith("/")
+                )
+
+                items.append(
+                    {
+                        "retailer": RETAILER,
+                        "order_id": order_id,
+                        "line_no": str(line_no),
+                        "order_date": receipt.get("transactionDate", ""),
+                        "retailer_item_id": item_number,
+                        "pod_id": "",
+                        "item_name": description,
+                        "upc": "",
+                        "category_id": stringify(item.get("itemDepartmentNumber")),
+                        "category": stringify(item.get("transDepartmentNumber")),
+                        "qty": stringify(item.get("unit")),
+                        "unit": stringify(item.get("itemIdentifier")),
+                        "unit_price": stringify(item.get("itemUnitPriceAmount")),
+                        "line_total": stringify(item.get("amount")),
+                        "picked_weight": "",
+                        "mvp_savings": "",
+                        "reward_savings": "",
+                        "coupon_savings": stringify(item.get("amount") if is_coupon else ""),
+                        "coupon_price": "",
+                        "image_url": "",
+                        "raw_order_path": raw_order_path.as_posix(),
+                        "is_discount_line": "true" if is_discount else "false",
+                        "is_coupon_line": "true" if is_coupon else "false",
+                    }
+                )
+
+    return orders, items
+
+
+def join_descriptions(*parts):
+    return " ".join(str(part).strip() for part in parts if part).strip()
+
+
+def compact_join(rows, field):
+    values = [str(row.get(field, "")).strip() for row in rows if row.get(field)]
+    return " | ".join(values)
+
+
+def is_discount_line(item):
+    amount = item.get("amount")
+    unit = item.get("unit")
+    description = join_descriptions(
+        item.get("itemDescription01"), item.get("itemDescription02")
+    )
+    try:
+        amount_val = float(amount)
+    except (TypeError, ValueError):
+        amount_val = 0.0
+    try:
+        unit_val = float(unit)
+    except (TypeError, ValueError):
+        unit_val = 0.0
+    return amount_val < 0 or unit_val < 0 or description.startswith("/")
+
+
+def stringify(value):
+    if value is None:
+        return ""
+    return str(value)
+
+
+def write_json(path, payload):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+
+def write_csv(path, rows, fieldnames):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+@click.command()
+@click.option(
+    "--outdir",
+    default="costco_output",
+    show_default=True,
+    help="Output directory for Costco raw and flattened files.",
+)
+@click.option(
+    "--document-type",
+    default="all",
+    show_default=True,
+    help="Summary document type.",
+)
+@click.option(
+    "--document-sub-type",
+    default="all",
+    show_default=True,
+    help="Summary document sub type.",
+)
+@click.option(
+    "--window-days",
+    default=92,
+    show_default=True,
+    type=int,
+    help="Maximum number of days to request per summary window.",
+)
+@click.option(
+    "--months-back",
+    default=3,
+    show_default=True,
+    type=int,
+    help="How many months of receipts to enumerate back from today.",
+)
+def main(outdir, document_type, document_sub_type, window_days, months_back):
+    outdir = Path(outdir)
+    raw_dir = outdir / "raw"
+    try:
+        session = build_session()
+    except Exception as exc:
+        raise click.ClickException(
+            f"failed to load Costco Firefox cookies: {exc}"
+        ) from exc
+    start_date, end_date = resolve_date_range(months_back)
+
+    summary_payload, request_metadata = fetch_summary_windows(
+        session,
+        start_date,
+        end_date,
+        document_type,
+        document_sub_type,
+        window_days,
+    )
+    write_json(raw_dir / "summary.json", summary_payload)
+    write_json(raw_dir / "summary_requests.json", request_metadata)
+    receipts = summary_receipts(summary_payload)
+
+    detail_payloads = []
+    for receipt in receipts:
+        barcode = receipt["transactionBarcode"]
+        receipt_id = receipt_key(receipt) or barcode
+        click.echo(f"fetching {barcode}")
+        detail_payload = graphql_post(
+            session,
+            DETAIL_QUERY,
+            {"barcode": barcode, "documentType": "warehouse"},
+        )
+        detail_payloads.append(detail_payload)
+        write_json(raw_dir / f"{receipt_id}.json", detail_payload)
+
+    orders, items = flatten_costco_data(summary_payload, detail_payloads, raw_dir)
+    write_csv(outdir / "orders.csv", orders, ORDER_FIELDS)
+    write_csv(outdir / "items.csv", items, ITEM_FIELDS)
+    click.echo(f"wrote {len(orders)} orders and {len(items)} item rows to {outdir}")
+
+
+if __name__ == "__main__":
+    main()

tests/test_canonical_layer.py

@@ -9,6 +9,7 @@ class CanonicalLayerTests(unittest.TestCase):
             {
                 "observed_product_id": "gobs_1",
                 "representative_upc": "111",
+                "representative_retailer_item_id": "11",
                 "representative_name_norm": "GALA APPLE",
                 "representative_brand": "SB",
                 "representative_variant": "",
@@ -17,10 +18,13 @@ class CanonicalLayerTests(unittest.TestCase):
                 "representative_pack_qty": "",
                 "representative_measure_type": "weight",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
             },
             {
                 "observed_product_id": "gobs_2",
                 "representative_upc": "111",
+                "representative_retailer_item_id": "12",
                 "representative_name_norm": "LARGE WHITE EGGS",
                 "representative_brand": "SB",
                 "representative_variant": "",
@@ -29,10 +33,13 @@ class CanonicalLayerTests(unittest.TestCase):
                 "representative_pack_qty": "18",
                 "representative_measure_type": "count",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
             },
             {
                 "observed_product_id": "gobs_3",
                 "representative_upc": "",
+                "representative_retailer_item_id": "21",
                 "representative_name_norm": "ROTINI",
                 "representative_brand": "",
                 "representative_variant": "",
@@ -41,10 +48,13 @@ class CanonicalLayerTests(unittest.TestCase):
                 "representative_pack_qty": "",
                 "representative_measure_type": "weight",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
             },
             {
                 "observed_product_id": "gobs_4",
                 "representative_upc": "",
+                "representative_retailer_item_id": "22",
                 "representative_name_norm": "ROTINI",
                 "representative_brand": "SB",
                 "representative_variant": "",
@@ -53,10 +63,13 @@ class CanonicalLayerTests(unittest.TestCase):
                 "representative_pack_qty": "",
                 "representative_measure_type": "weight",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
             },
             {
                 "observed_product_id": "gobs_5",
                 "representative_upc": "",
+                "representative_retailer_item_id": "99",
                 "representative_name_norm": "GL BAG CHARGE",
                 "representative_brand": "",
                 "representative_variant": "",
@@ -65,6 +78,8 @@ class CanonicalLayerTests(unittest.TestCase):
                 "representative_pack_qty": "",
                 "representative_measure_type": "each",
                 "is_fee": "true",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
             },
         ]
 

tests/test_costco_pipeline.py

@@ -0,0 +1,439 @@
+import csv
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import enrich_costco
+import scrape_costco
+import validate_cross_retailer_flow
+
+
+class CostcoPipelineTests(unittest.TestCase):
+    def test_resolve_date_range_uses_months_back(self):
+        start_date, end_date = scrape_costco.resolve_date_range(
+            3, today=scrape_costco.parse_cli_date("3/16/2026")
+        )
+
+        self.assertEqual("12/16/2025", start_date)
+        self.assertEqual("3/16/2026", end_date)
+
+    def test_build_date_windows_splits_long_ranges(self):
+        windows = scrape_costco.build_date_windows("1/01/2026", "6/30/2026", 92)
+
+        self.assertEqual(
+            [
+                {"startDate": "1/01/2026", "endDate": "4/02/2026"},
+                {"startDate": "4/03/2026", "endDate": "6/30/2026"},
+            ],
+            windows,
+        )
+
+    def test_fetch_summary_windows_records_metadata_and_warns_on_mismatch(self):
+        payloads = [
+            {
+                "data": {
+                    "receiptsWithCounts": {
+                        "inWarehouse": 2,
+                        "gasStation": 0,
+                        "carWash": 0,
+                        "gasAndCarWash": 0,
+                        "receipts": [
+                            {
+                                "transactionBarcode": "abc",
+                                "receiptType": "In-Warehouse",
+                            }
+                        ],
+                    }
+                }
+            },
+            {
+                "data": {
+                    "receiptsWithCounts": {
+                        "inWarehouse": 1,
+                        "gasStation": 0,
+                        "carWash": 0,
+                        "gasAndCarWash": 0,
+                        "receipts": [
+                            {
+                                "transactionBarcode": "def",
+                                "receiptType": "In-Warehouse",
+                            }
+                        ],
+                    }
+                }
+            },
+        ]
+
+        with mock.patch.object(
+            scrape_costco, "graphql_post", side_effect=payloads
+        ) as mocked_post, mock.patch.object(scrape_costco.click, "echo") as mocked_echo:
+            summary_payload, metadata = scrape_costco.fetch_summary_windows(
+                session=object(),
+                start_date="1/01/2026",
+                end_date="6/30/2026",
+                document_type="all",
+                document_sub_type="all",
+                window_days=92,
+            )
+
+        self.assertEqual(2, mocked_post.call_count)
+        self.assertEqual(2, len(metadata))
+        self.assertTrue(metadata[0]["countMismatch"])
+        self.assertFalse(metadata[1]["countMismatch"])
+        self.assertEqual("1/01/2026", metadata[0]["startDate"])
+        self.assertEqual("4/03/2026", metadata[1]["startDate"])
+        self.assertEqual(
+            ["abc", "def"],
+            [
+                row["transactionBarcode"]
+                for row in scrape_costco.summary_receipts(summary_payload)
+            ],
+        )
+        mocked_echo.assert_called_once()
+        warning_text = mocked_echo.call_args.args[0]
+        self.assertIn("warning: summary count mismatch", warning_text)
+
+    def test_flatten_costco_data_preserves_discount_rows(self):
+        summary_payload = {
+            "data": {
+                "receiptsWithCounts": {
+                    "receipts": [
+                        {
+                            "transactionBarcode": "abc",
+                            "tenderArray": [{"tenderDescription": "VISA"}],
+                            "couponArray": [{"upcnumberCoupon": "2100003746641"}],
+                        }
+                    ]
+                }
+            }
+        }
+        detail_payloads = [
+            {
+                "data": {
+                    "receiptsWithCounts": {
+                        "receipts": [
+                            {
+                                "transactionBarcode": "abc",
+                                "transactionDate": "2026-03-12",
+                                "receiptType": "In-Warehouse",
+                                "total": 10.0,
+                                "totalItemCount": 2,
+                                "instantSavings": 5.0,
+                                "warehouseName": "MT VERNON",
+                                "warehouseNumber": 1115,
+                                "warehouseAddress1": "7940 RICHMOND HWY",
+                                "warehouseCity": "ALEXANDRIA",
+                                "warehouseState": "VA",
+                                "warehousePostalCode": "22306",
+                                "itemArray": [
+                                    {
+                                        "itemNumber": "4873222",
+                                        "itemDescription01": "ALL F&C",
+                                        "itemDescription02": "200OZ 160LOADS P104",
+                                        "itemDepartmentNumber": 14,
+                                        "transDepartmentNumber": 14,
+                                        "unit": 1,
+                                        "itemIdentifier": "E",
+                                        "amount": 19.99,
+                                        "itemUnitPriceAmount": 19.99,
+                                    },
+                                    {
+                                        "itemNumber": "374664",
+                                        "itemDescription01": "/ 4873222",
+                                        "itemDescription02": None,
+                                        "itemDepartmentNumber": 14,
+                                        "transDepartmentNumber": 14,
+                                        "unit": -1,
+                                        "itemIdentifier": None,
+                                        "amount": -5,
+                                        "itemUnitPriceAmount": 0,
+                                    },
+                                ],
+                            }
+                        ]
+                    }
+                }
+            }
+        ]
+
+        orders, items = scrape_costco.flatten_costco_data(
+            summary_payload, detail_payloads, Path("costco_output/raw")
+        )
+
+        self.assertEqual(1, len(orders))
+        self.assertEqual(2, len(items))
+        self.assertEqual("false", items[0]["is_discount_line"])
+        self.assertEqual("true", items[1]["is_discount_line"])
+        self.assertEqual("true", items[1]["is_coupon_line"])
+
+    def test_flatten_costco_data_uses_composite_summary_lookup_key(self):
+        summary_payload = {
+            "data": {
+                "receiptsWithCounts": {
+                    "receipts": [
+                        {
+                            "transactionBarcode": "dup",
+                            "transactionDateTime": "2026-03-12T16:16:00",
+                            "tenderArray": [{"tenderDescription": "VISA"}],
+                            "couponArray": [{"upcnumberCoupon": "111"}],
+                        },
+                        {
+                            "transactionBarcode": "dup",
+                            "transactionDateTime": "2026-02-14T16:25:00",
+                            "tenderArray": [{"tenderDescription": "MASTERCARD"}],
+                            "couponArray": [],
+                        },
+                    ]
+                }
+            }
+        }
+        detail_payloads = [
+            {
+                "data": {
+                    "receiptsWithCounts": {
+                        "receipts": [
+                            {
+                                "transactionBarcode": "dup",
+                                "transactionDateTime": "2026-03-12T16:16:00",
+                                "transactionDate": "2026-03-12",
+                                "receiptType": "In-Warehouse",
+                                "total": 10.0,
+                                "totalItemCount": 1,
+                                "instantSavings": 5.0,
+                                "warehouseName": "MT VERNON",
+                                "warehouseNumber": 1115,
+                                "warehouseAddress1": "7940 RICHMOND HWY",
+                                "warehouseCity": "ALEXANDRIA",
+                                "warehouseState": "VA",
+                                "warehousePostalCode": "22306",
+                                "itemArray": [
+                                    {
+                                        "itemNumber": "111",
+                                        "itemDescription01": "/ 111",
+                                        "itemDescription02": None,
+                                        "itemDepartmentNumber": 14,
+                                        "transDepartmentNumber": 14,
+                                        "unit": -1,
+                                        "itemIdentifier": None,
+                                        "amount": -5,
+                                        "itemUnitPriceAmount": 0,
+                                    }
+                                ],
+                            }
+                        ]
+                    }
+                }
+            }
+        ]
+
+        orders, items = scrape_costco.flatten_costco_data(
+            summary_payload, detail_payloads, Path("costco_output/raw")
+        )
+
+        self.assertEqual("VISA", orders[0]["payment_method"])
+        self.assertEqual("true", items[0]["is_coupon_line"])
+        self.assertIn("dup::2026-03-12T16:16:00.json", items[0]["raw_order_path"])
+
+    def test_costco_enricher_parses_size_pack_and_discount(self):
+        row = enrich_costco.parse_costco_item(
+            order_id="abc",
+            order_date="2026-03-12",
+            raw_path=Path("costco_output/raw/abc.json"),
+            line_no=1,
+            item={
+                "itemNumber": "60357",
+                "itemDescription01": "MIXED PEPPER",
+                "itemDescription02": "6-PACK",
+                "itemDepartmentNumber": 65,
+                "transDepartmentNumber": 65,
+                "unit": 1,
+                "itemIdentifier": "E",
+                "amount": 7.49,
+                "itemUnitPriceAmount": 7.49,
+            },
+        )
+        self.assertEqual("60357", row["retailer_item_id"])
+        self.assertEqual("MIXED PEPPER", row["item_name_norm"])
+        self.assertEqual("6", row["pack_qty"])
+        self.assertEqual("count", row["measure_type"])
+
+        discount = enrich_costco.parse_costco_item(
+            order_id="abc",
+            order_date="2026-03-12",
+            raw_path=Path("costco_output/raw/abc.json"),
+            line_no=2,
+            item={
+                "itemNumber": "374664",
+                "itemDescription01": "/ 4873222",
+                "itemDescription02": None,
+                "itemDepartmentNumber": 14,
+                "transDepartmentNumber": 14,
+                "unit": -1,
+                "itemIdentifier": None,
+                "amount": -5,
+                "itemUnitPriceAmount": 0,
+            },
+        )
+        self.assertEqual("true", discount["is_discount_line"])
+        self.assertEqual("true", discount["is_coupon_line"])
+
+    def test_cross_retailer_validation_writes_proof_example(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            giant_csv = Path(tmpdir) / "giant_items_enriched.csv"
+            costco_csv = Path(tmpdir) / "costco_items_enriched.csv"
+            outdir = Path(tmpdir) / "combined"
+
+            fieldnames = enrich_costco.OUTPUT_FIELDS
+            giant_row = {field: "" for field in fieldnames}
+            giant_row.update(
+                {
+                    "retailer": "giant",
+                    "order_id": "g1",
+                    "line_no": "1",
+                    "order_date": "2026-03-01",
+                    "retailer_item_id": "100",
+                    "item_name": "FRESH BANANA",
+                    "item_name_norm": "BANANA",
+                    "upc": "4011",
+                    "measure_type": "weight",
+                    "is_store_brand": "false",
+                    "is_fee": "false",
+                    "is_discount_line": "false",
+                    "is_coupon_line": "false",
+                    "line_total": "1.29",
+                }
+            )
+            costco_row = {field: "" for field in fieldnames}
+            costco_row.update(
+                {
+                    "retailer": "costco",
+                    "order_id": "c1",
+                    "line_no": "1",
+                    "order_date": "2026-03-12",
+                    "retailer_item_id": "30669",
+                    "item_name": "BANANAS 3 LB / 1.36 KG",
+                    "item_name_norm": "BANANA",
+                    "upc": "",
+                    "size_value": "3",
+                    "size_unit": "lb",
+                    "measure_type": "weight",
+                    "is_store_brand": "false",
+                    "is_fee": "false",
+                    "is_discount_line": "false",
+                    "is_coupon_line": "false",
+                    "line_total": "2.98",
+                }
+            )
+
+            with giant_csv.open("w", newline="", encoding="utf-8") as handle:
+                writer = csv.DictWriter(handle, fieldnames=fieldnames)
+                writer.writeheader()
+                writer.writerow(giant_row)
+            with costco_csv.open("w", newline="", encoding="utf-8") as handle:
+                writer = csv.DictWriter(handle, fieldnames=fieldnames)
+                writer.writeheader()
+                writer.writerow(costco_row)
+
+            validate_cross_retailer_flow.main.callback(
+                giant_items_enriched_csv=str(giant_csv),
+                costco_items_enriched_csv=str(costco_csv),
+                outdir=str(outdir),
+            )
+
+            proof_path = outdir / "proof_examples.csv"
+            self.assertTrue(proof_path.exists())
+            with proof_path.open(newline="", encoding="utf-8") as handle:
+                rows = list(csv.DictReader(handle))
+            self.assertEqual(1, len(rows))
+            self.assertEqual("banana", rows[0]["proof_name"])
+
+    def test_main_writes_summary_request_metadata(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            outdir = Path(tmpdir) / "costco_output"
+            summary_payload = {
+                "data": {
+                    "receiptsWithCounts": {
+                        "inWarehouse": 1,
+                        "gasStation": 0,
+                        "carWash": 0,
+                        "gasAndCarWash": 0,
+                        "receipts": [
+                            {
+                                "transactionBarcode": "abc",
+                                "receiptType": "In-Warehouse",
+                                "tenderArray": [],
+                                "couponArray": [],
+                            }
+                        ],
+                    }
+                }
+            }
+            detail_payload = {
+                "data": {
+                    "receiptsWithCounts": {
+                        "receipts": [
+                            {
+                                "transactionBarcode": "abc",
+                                "transactionDate": "2026-03-12",
+                                "receiptType": "In-Warehouse",
+                                "total": 10.0,
+                                "totalItemCount": 1,
+                                "instantSavings": 0,
+                                "warehouseName": "MT VERNON",
+                                "warehouseNumber": 1115,
+                                "warehouseAddress1": "7940 RICHMOND HWY",
+                                "warehouseCity": "ALEXANDRIA",
+                                "warehouseState": "VA",
+                                "warehousePostalCode": "22306",
+                                "itemArray": [],
+                            }
+                        ]
+                    }
+                }
+            }
+            metadata = [
+                {
+                    "startDate": "1/01/2026",
+                    "endDate": "3/31/2026",
+                    "text": "custom",
+                    "documentType": "all",
+                    "documentSubType": "all",
+                    "returnedReceipts": 1,
+                    "returnedInWarehouseReceipts": 1,
+                    "inWarehouse": 1,
+                    "gasStation": 0,
+                    "carWash": 0,
+                    "gasAndCarWash": 0,
+                    "countMismatch": False,
+                }
+            ]
+
+            with mock.patch.object(
+                scrape_costco, "build_session", return_value=object()
+            ), mock.patch.object(
+                scrape_costco,
+                "fetch_summary_windows",
+                return_value=(summary_payload, metadata),
+            ), mock.patch.object(
+                scrape_costco,
+                "graphql_post",
+                return_value=detail_payload,
+            ):
+                scrape_costco.main.callback(
+                    outdir=str(outdir),
+                    document_type="all",
+                    document_sub_type="all",
+                    window_days=92,
+                    months_back=3,
+                )
+
+            metadata_path = outdir / "raw" / "summary_requests.json"
+            self.assertTrue(metadata_path.exists())
+            saved_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+            self.assertEqual(metadata, saved_metadata)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_enrich_giant.py

@@ -177,6 +177,7 @@ class EnrichGiantTests(unittest.TestCase):
             self.assertEqual("PEPSI", rows[0]["item_name_norm"])
             self.assertEqual("6", rows[0]["pack_qty"])
             self.assertEqual("7.5", rows[0]["size_value"])
+            self.assertEqual("10", rows[0]["retailer_item_id"])
             self.assertEqual("true", rows[1]["is_store_brand"])
 
             with output_csv.open(newline="", encoding="utf-8") as handle:

tests/test_observed_products.py

@@ -13,6 +13,7 @@ class ObservedProductTests(unittest.TestCase):
                 "order_date": "2026-01-01",
                 "item_name": "SB GALA APPLE 5LB",
                 "item_name_norm": "GALA APPLE",
+                "retailer_item_id": "11",
                 "upc": "111",
                 "brand_guess": "SB",
                 "variant": "",
@@ -23,6 +24,8 @@ class ObservedProductTests(unittest.TestCase):
                 "image_url": "https://example.test/a.jpg",
                 "is_store_brand": "true",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
                 "line_total": "7.99",
             },
             {
@@ -32,6 +35,7 @@ class ObservedProductTests(unittest.TestCase):
                 "order_date": "2026-01-10",
                 "item_name": "SB GALA APPLE 5 LB",
                 "item_name_norm": "GALA APPLE",
+                "retailer_item_id": "11",
                 "upc": "111",
                 "brand_guess": "SB",
                 "variant": "",
@@ -42,6 +46,8 @@ class ObservedProductTests(unittest.TestCase):
                 "image_url": "",
                 "is_store_brand": "true",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
                 "line_total": "8.49",
             },
         ]
@@ -52,6 +58,7 @@ class ObservedProductTests(unittest.TestCase):
         self.assertEqual("2", observed[0]["times_seen"])
         self.assertEqual("2026-01-01", observed[0]["first_seen_date"])
         self.assertEqual("2026-01-10", observed[0]["last_seen_date"])
+        self.assertEqual("11", observed[0]["representative_retailer_item_id"])
         self.assertEqual("111", observed[0]["representative_upc"])
         self.assertIn("SB GALA APPLE 5LB", observed[0]["raw_name_examples"])
 

tests/test_review_queue.py

@@ -20,6 +20,8 @@ class ReviewQueueTests(unittest.TestCase):
                 "distinct_item_names_count": "2",
                 "distinct_upcs_count": "1",
                 "is_fee": "false",
+                "is_discount_line": "false",
+                "is_coupon_line": "false",
             }
         ]
         item_rows = [
@@ -64,6 +66,7 @@ class ReviewQueueTests(unittest.TestCase):
                     "observed_product_id": "gobs_1",
                     "retailer": "giant",
                     "observed_key": "giant|upc=111|name=GALA APPLE",
+                    "representative_retailer_item_id": "11",
                     "representative_upc": "111",
                     "representative_item_name": "SB GALA APPLE 5LB",
                     "representative_name_norm": "GALA APPLE",
@@ -76,6 +79,8 @@ class ReviewQueueTests(unittest.TestCase):
                     "representative_image_url": "",
                     "is_store_brand": "true",
                     "is_fee": "false",
+                    "is_discount_line": "false",
+                    "is_coupon_line": "false",
                     "first_seen_date": "2026-01-01",
                     "last_seen_date": "2026-01-10",
                     "times_seen": "2",
@@ -85,6 +90,7 @@ class ReviewQueueTests(unittest.TestCase):
                     "normalized_name_examples": "GALA APPLE",
                     "example_prices": "7.99 | 8.49",
                     "distinct_item_names_count": "2",
+                    "distinct_retailer_item_ids_count": "1",
                     "distinct_upcs_count": "1",
                 }
             ]
@@ -95,6 +101,7 @@ class ReviewQueueTests(unittest.TestCase):
                     "line_no": "1",
                     "item_name": "SB GALA APPLE 5LB",
                     "item_name_norm": "GALA APPLE",
+                    "retailer_item_id": "11",
                     "upc": "111",
                     "size_value": "5",
                     "size_unit": "lb",
@@ -102,6 +109,8 @@ class ReviewQueueTests(unittest.TestCase):
                     "measure_type": "weight",
                     "is_store_brand": "true",
                     "is_fee": "false",
+                    "is_discount_line": "false",
+                    "is_coupon_line": "false",
                     "line_total": "7.99",
                 }
             ]

validate_cross_retailer_flow.py

@@ -0,0 +1,154 @@
+import json
+from pathlib import Path
+
+import click
+
+import build_canonical_layer
+import build_observed_products
+from layer_helpers import stable_id, write_csv_rows
+
+
+PROOF_FIELDS = [
+    "proof_name",
+    "canonical_product_id",
+    "giant_observed_product_id",
+    "costco_observed_product_id",
+    "giant_example_item",
+    "costco_example_item",
+    "notes",
+]
+
+
+def read_rows(path):
+    import csv
+
+    with Path(path).open(newline="", encoding="utf-8") as handle:
+        return list(csv.DictReader(handle))
+
+
+def find_proof_pair(observed_rows):
+    giant = None
+    costco = None
+    for row in observed_rows:
+        if row["retailer"] == "giant" and row["representative_name_norm"] == "BANANA":
+            giant = row
+        if row["retailer"] == "costco" and row["representative_name_norm"] == "BANANA":
+            costco = row
+    return giant, costco
+
+
+def merge_proof_pair(canonical_rows, link_rows, giant_row, costco_row):
+    if not giant_row or not costco_row:
+        return canonical_rows, link_rows, []
+
+    proof_canonical_id = stable_id("gcan", "proof|banana")
+    link_rows = [
+        row
+        for row in link_rows
+        if row["observed_product_id"]
+        not in {giant_row["observed_product_id"], costco_row["observed_product_id"]}
+    ]
+    canonical_rows = [
+        row
+        for row in canonical_rows
+        if row["canonical_product_id"] != proof_canonical_id
+    ]
+    canonical_rows.append(
+        {
+            "canonical_product_id": proof_canonical_id,
+            "canonical_name": "BANANA",
+            "product_type": "banana",
+            "brand": "",
+            "variant": "",
+            "size_value": "",
+            "size_unit": "",
+            "pack_qty": "",
+            "measure_type": "weight",
+            "normalized_quantity": "",
+            "normalized_quantity_unit": "",
+            "notes": "manual proof merge for cross-retailer validation",
+            "created_at": "",
+            "updated_at": "",
+        }
+    )
+    for observed_row in [giant_row, costco_row]:
+        link_rows.append(
+            {
+                "observed_product_id": observed_row["observed_product_id"],
+                "canonical_product_id": proof_canonical_id,
+                "link_method": "manual_proof_merge",
+                "link_confidence": "medium",
+                "review_status": "",
+                "reviewed_by": "",
+                "reviewed_at": "",
+                "link_notes": "cross-retailer validation proof",
+            }
+        )
+
+    proof_rows = [
+        {
+            "proof_name": "banana",
+            "canonical_product_id": proof_canonical_id,
+            "giant_observed_product_id": giant_row["observed_product_id"],
+            "costco_observed_product_id": costco_row["observed_product_id"],
+            "giant_example_item": giant_row["example_item_name"],
+            "costco_example_item": costco_row["example_item_name"],
+            "notes": "BANANA proof pair built from Giant and Costco enriched rows",
+        }
+    ]
+    return canonical_rows, link_rows, proof_rows
+
+
+@click.command()
+@click.option(
+    "--giant-items-enriched-csv",
+    default="giant_output/items_enriched.csv",
+    show_default=True,
+)
+@click.option(
+    "--costco-items-enriched-csv",
+    default="costco_output/items_enriched.csv",
+    show_default=True,
+)
+@click.option(
+    "--outdir",
+    default="combined_output",
+    show_default=True,
+)
+def main(giant_items_enriched_csv, costco_items_enriched_csv, outdir):
+    outdir = Path(outdir)
+    rows = read_rows(giant_items_enriched_csv) + read_rows(costco_items_enriched_csv)
+    observed_rows = build_observed_products.build_observed_products(rows)
+    canonical_rows, link_rows = build_canonical_layer.build_canonical_layer(observed_rows)
+    giant_row, costco_row = find_proof_pair(observed_rows)
+    if not giant_row or not costco_row:
+        raise click.ClickException(
+            "could not find BANANA proof pair across Giant and Costco observed products"
+        )
+    canonical_rows, link_rows, proof_rows = merge_proof_pair(
+        canonical_rows, link_rows, giant_row, costco_row
+    )
+
+    write_csv_rows(
+        outdir / "products_observed.csv",
+        observed_rows,
+        build_observed_products.OUTPUT_FIELDS,
+    )
+    write_csv_rows(
+        outdir / "products_canonical.csv",
+        canonical_rows,
+        build_canonical_layer.CANONICAL_FIELDS,
+    )
+    write_csv_rows(
+        outdir / "product_links.csv",
+        link_rows,
+        build_canonical_layer.LINK_FIELDS,
+    )
+    write_csv_rows(outdir / "proof_examples.csv", proof_rows, PROOF_FIELDS)
+    click.echo(
+        f"wrote combined outputs to {outdir} using {len(observed_rows)} observed rows"
+    )
+
+
+if __name__ == "__main__":
+    main()