ben/scrape-giant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated to use .env, then pull idToken and clientID

Browse Source
This commit is contained in:
ben
2026-03-16 17:17:20 -04:00
parent b0d4044dac
commit 9a985bf98d
3 changed files with 45 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
scrape_costco.py
View File

@@ -216,12 +216,13 @@ ITEM_FIELDS = [
]
COSTCO_STORAGE_ORIGIN = "costco.com"
COSTCO_AUTH_STORAGE_KEY = "costco-x-authorization"
COSTCO_HEADERS_BLOB_KEY = "headers"
COSTCO_ID_TOKEN_STORAGE_KEY = "idToken"
COSTCO_CLIENT_ID_STORAGE_KEY = "clientID"
def load_config():
load_dotenv()
return {
"authorization": os.getenv("COSTCO_X_AUTHORIZATION", "").strip(),
"client_id": os.getenv("COSTCO_X_WCS_CLIENTID", "").strip(),
"client_identifier": os.getenv("COSTCO_CLIENT_IDENTIFIER", "").strip(),
}
@@ -244,33 +245,31 @@ def build_headers(auth_headers):
return headers
def load_costco_browser_headers(profile_dir, client_id, client_identifier):
def load_costco_browser_headers(profile_dir, authorization, client_id, client_identifier):
local_storage = read_firefox_local_storage(profile_dir, COSTCO_STORAGE_ORIGIN)
webapps_store = read_firefox_webapps_store(profile_dir, COSTCO_STORAGE_ORIGIN)
auth_token = (
local_storage.get(COSTCO_AUTH_STORAGE_KEY, "").strip()
or webapps_store.get(COSTCO_AUTH_STORAGE_KEY, "").strip()
auth_header = authorization.strip() if authorization else ""
if client_id:
client_id = client_id.strip()
if client_identifier:
client_identifier = client_identifier.strip()
if not auth_header:
id_token = (
local_storage.get(COSTCO_ID_TOKEN_STORAGE_KEY, "").strip()
or webapps_store.get(COSTCO_ID_TOKEN_STORAGE_KEY, "").strip()
)
if id_token:
auth_header = f"Bearer {id_token}"
client_id = client_id or (
local_storage.get(COSTCO_CLIENT_ID_STORAGE_KEY, "").strip()
or webapps_store.get(COSTCO_CLIENT_ID_STORAGE_KEY, "").strip()
)
if not auth_token:
header_blob = (
local_storage.get(COSTCO_HEADERS_BLOB_KEY, "").strip()
or webapps_store.get(COSTCO_HEADERS_BLOB_KEY, "").strip()
)
if header_blob:
try:
blob_data = json.loads(header_blob)
except json.JSONDecodeError:
blob_data = {}
auth_token = str(blob_data.get(COSTCO_AUTH_STORAGE_KEY, "")).strip()
client_id = client_id or str(blob_data.get("costco-x-wcs-clientId", "")).strip()
client_identifier = client_identifier or str(
blob_data.get("client-identifier", "")
).strip()
if not auth_token:
if not auth_header:
raise click.ClickException(
"could not find Costco auth token in Firefox session storage"
"could not find Costco auth token; set COSTCO_X_AUTHORIZATION or load Firefox idToken"
)
if not client_id or not client_identifier:
raise click.ClickException(
@@ -278,7 +277,7 @@ def load_costco_browser_headers(profile_dir, client_id, client_identifier):
)
return {
"costco-x-authorization": auth_token,
"costco-x-authorization": auth_header,
"costco-x-wcs-clientId": client_id,
"client-identifier": client_identifier,
}
@@ -666,6 +665,7 @@ def main(
auth_headers = load_costco_browser_headers(
profile_dir,
authorization=config["authorization"],
client_id=config["client_id"],
client_identifier=config["client_identifier"],
)