troubleshooting costco header extraction

scrape_costco.py

@@ -1,3 +1,4 @@
+import os
 import csv
 import json
 import time
@@ -5,7 +6,7 @@ import re
 from calendar import monthrange
 from datetime import datetime, timedelta
 from pathlib import Path
-
+from dotenv import load_dotenv
 import click
 from curl_cffi import requests
 
@@ -23,7 +24,7 @@ query receiptsWithCounts($startDate: String!, $endDate: String!, $documentType:
     gasAndCarWash
     receipts {
       warehouseName
-      receiptType
+      receiptType 
       documentType
       transactionDateTime
       transactionBarcode
@@ -225,11 +226,11 @@ def build_headers(auth_headers):
     headers.update(auth_headers)
     return headers
 
-def build_session(profile_dir=None):
-    retailer_session = load_costco_session(profile_dir=profile_dir)
+def build_session(retailer_session):
     session = requests.Session()
     session.cookies.update(retailer_session.cookies)
-    session.headers.update(build_headers(retailer_session.headers))
+    session.headers.update(build_headers())
+    session.headers.update(retailer_session.headers)
     return session
 
 
@@ -593,23 +594,28 @@ def main(
 ):
     outdir = Path(outdir)
     raw_dir = outdir / "raw"
-    try:
-        session = build_session(profile_dir=firefox_profile_dir)
-    except Exception as exc:
-        if firefox_profile_dir:
-            raise click.ClickException(
-                f"failed to load Costco browser session: {exc}"
-            ) from exc
-        prompted_profile = click.prompt(
-            "Firefox profile dir",
-            type=click.Path(exists=True, file_okay=False, path_type=Path),
+    if firefox_profile_dir is None:
+        firefox_profile_dir = next(
+            (Path(os.getenv("APPDATA")) / "Mozilla" / "Firefox" / "Profiles").iterdir()
         )
-        try:
-            session = build_session(profile_dir=prompted_profile)
-        except Exception as prompt_exc:
-            raise click.ClickException(
-                f"failed to load Costco browser session: {prompt_exc}"
-            ) from prompt_exc
+    try:
+        retailer_session = load_costco_session(
+            browser="firefox",
+            profile_dir=firefox_profile_dir,
+        )
+        click.echo(
+            "session bootstrap: "
+            f"cookies={bool(retailer_session.cookies)}, "
+            f"authorization={'costco-x-authorization' in retailer_session.headers}, "
+            f"client_id={'costco-x-wcs-clientId' in retailer_session.headers}, "
+            f"client_identifier={'client-identifier' in retailer_session.headers}"
+        )
+        session = build_session(retailer_session)
+    except Exception as exc:
+        raise click.ClickException(
+            f"failed to load Costco browser session: {exc}"
+        ) from exc
+
     start_date, end_date = resolve_date_range(months_back)
 
     summary_payload, request_metadata = fetch_summary_windows(
@@ -645,4 +651,5 @@ def main(
 
 if __name__ == "__main__":
     main()
+