Refactor retailer collection entrypoints

scrape_giant.py

@@ -13,8 +13,10 @@ from browser_session import find_firefox_profile_dir, load_firefox_cookies
 
 BASE = "https://giantfood.com"
 ACCOUNT_PAGE = f"{BASE}/account/history/invoice/in-store"
+RETAILER = "giant"
 
 ORDER_FIELDS = [
+    "retailer",
     "order_id",
     "order_date",
     "delivery_date",
@@ -33,12 +35,16 @@ ORDER_FIELDS = [
     "store_zipcode",
     "refund_order",
     "ebt_order",
+    "raw_history_path",
+    "raw_order_path",
 ]
 
 ITEM_FIELDS = [
+    "retailer",
     "order_id",
     "order_date",
     "line_no",
+    "retailer_item_id",
     "pod_id",
     "item_name",
     "upc",
@@ -53,6 +59,10 @@ ITEM_FIELDS = [
     "reward_savings",
     "coupon_savings",
     "coupon_price",
+    "image_url",
+    "raw_order_path",
+    "is_discount_line",
+    "is_coupon_line",
 ]
 
 
@@ -130,18 +140,21 @@ def get_order_detail(session, user_id, order_id):
     return response.json()
 
 
-def flatten_orders(history, details):
+def flatten_orders(history, details, history_path=None, raw_dir=None):
     orders = []
     items = []
     history_lookup = {record["orderId"]: record for record in history.get("records", [])}
+    history_path_value = history_path.as_posix() if history_path else ""
 
     for detail in details:
         order_id = str(detail["orderId"])
         history_row = history_lookup.get(detail["orderId"], {})
         pickup = detail.get("pup", {})
+        raw_order_path = (raw_dir / f"{order_id}.json").as_posix() if raw_dir else ""
 
         orders.append(
             {
+                "retailer": RETAILER,
                 "order_id": order_id,
                 "order_date": detail.get("orderDate"),
                 "delivery_date": detail.get("deliveryDate"),
@@ -160,15 +173,19 @@ def flatten_orders(history, details):
                 "store_zipcode": pickup.get("storeZipcode"),
                 "refund_order": detail.get("refundOrder"),
                 "ebt_order": detail.get("ebtOrder"),
+                "raw_history_path": history_path_value,
+                "raw_order_path": raw_order_path,
             }
         )
 
         for line_no, item in enumerate(detail.get("items", []), start=1):
             items.append(
                 {
+                    "retailer": RETAILER,
                     "order_id": order_id,
                     "order_date": detail.get("orderDate"),
                     "line_no": str(line_no),
+                    "retailer_item_id": "",
                     "pod_id": item.get("podId"),
                     "item_name": item.get("itemName"),
                     "upc": item.get("primUpcCd"),
@@ -183,6 +200,10 @@ def flatten_orders(history, details):
                     "reward_savings": item.get("rewardSavings"),
                     "coupon_savings": item.get("couponSavings"),
                     "coupon_price": item.get("couponPrice"),
+                    "image_url": "",
+                    "raw_order_path": raw_order_path,
+                    "is_discount_line": "false",
+                    "is_coupon_line": "false",
                 }
             )
 
@@ -269,6 +290,18 @@ def write_json(path, payload):
     help="Delay between order detail requests.",
 )
 def main(user_id, loyalty, outdir, sleep_seconds):
+    click.echo("legacy entrypoint: prefer collect_giant_web.py for data-model outputs")
+    run_collection(user_id, loyalty, outdir, sleep_seconds)
+
+
+def run_collection(
+    user_id,
+    loyalty,
+    outdir,
+    sleep_seconds,
+    orders_filename="orders.csv",
+    items_filename="items.csv",
+):
     config = load_config()
     user_id = user_id or config["user_id"] or click.prompt("Giant user id", type=str)
     loyalty = loyalty or config["loyalty"] or click.prompt(
@@ -279,13 +312,14 @@ def main(user_id, loyalty, outdir, sleep_seconds):
     rawdir = outdir / "raw"
     rawdir.mkdir(parents=True, exist_ok=True)
 
-    orders_csv = outdir / "orders.csv"
-    items_csv = outdir / "items.csv"
+    orders_csv = outdir / orders_filename
+    items_csv = outdir / items_filename
     existing_order_ids = read_existing_order_ids(orders_csv)
 
     session = build_session()
     history = get_history(session, user_id, loyalty)
-    write_json(rawdir / "history.json", history)
+    history_path = rawdir / "history.json"
+    write_json(history_path, history)
 
     records = history.get("records", [])
     click.echo(f"history returned {len(records)} visits; Giant exposes only the most recent 50")
@@ -310,7 +344,7 @@ def main(user_id, loyalty, outdir, sleep_seconds):
         if index < len(unseen_records):
             time.sleep(sleep_seconds)
 
-    orders, items = flatten_orders(history, details)
+    orders, items = flatten_orders(history, details, history_path=history_path, raw_dir=rawdir)
     merged_orders = append_dedup(
         orders_csv,
         orders,