534 lines
22 KiB
Python
534 lines
22 KiB
Python
import csv
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import enrich_costco
|
|
import scrape_costco
|
|
|
|
|
|
class CostcoPipelineTests(unittest.TestCase):
|
|
def test_resolve_date_range_uses_months_back(self):
|
|
start_date, end_date = scrape_costco.resolve_date_range(
|
|
3, today=scrape_costco.parse_cli_date("3/16/2026")
|
|
)
|
|
|
|
self.assertEqual("12/16/2025", start_date)
|
|
self.assertEqual("3/16/2026", end_date)
|
|
|
|
def test_build_date_windows_splits_long_ranges(self):
|
|
windows = scrape_costco.build_date_windows("1/01/2026", "6/30/2026", 92)
|
|
|
|
self.assertEqual(
|
|
[
|
|
{"startDate": "1/01/2026", "endDate": "4/02/2026"},
|
|
{"startDate": "4/03/2026", "endDate": "6/30/2026"},
|
|
],
|
|
windows,
|
|
)
|
|
|
|
def test_fetch_summary_windows_records_metadata_and_warns_on_mismatch(self):
|
|
payloads = [
|
|
{
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"inWarehouse": 2,
|
|
"gasStation": 0,
|
|
"carWash": 0,
|
|
"gasAndCarWash": 0,
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "abc",
|
|
"receiptType": "In-Warehouse",
|
|
}
|
|
],
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"inWarehouse": 1,
|
|
"gasStation": 0,
|
|
"carWash": 0,
|
|
"gasAndCarWash": 0,
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "def",
|
|
"receiptType": "In-Warehouse",
|
|
}
|
|
],
|
|
}
|
|
}
|
|
},
|
|
]
|
|
|
|
with mock.patch.object(
|
|
scrape_costco, "graphql_post", side_effect=payloads
|
|
) as mocked_post, mock.patch.object(scrape_costco.click, "echo") as mocked_echo:
|
|
summary_payload, metadata = scrape_costco.fetch_summary_windows(
|
|
session=object(),
|
|
start_date="1/01/2026",
|
|
end_date="6/30/2026",
|
|
document_type="all",
|
|
document_sub_type="all",
|
|
window_days=92,
|
|
)
|
|
|
|
self.assertEqual(2, mocked_post.call_count)
|
|
self.assertEqual(2, len(metadata))
|
|
self.assertTrue(metadata[0]["countMismatch"])
|
|
self.assertFalse(metadata[1]["countMismatch"])
|
|
self.assertEqual("1/01/2026", metadata[0]["startDate"])
|
|
self.assertEqual("4/03/2026", metadata[1]["startDate"])
|
|
self.assertEqual(
|
|
["abc", "def"],
|
|
[
|
|
row["transactionBarcode"]
|
|
for row in scrape_costco.summary_receipts(summary_payload)
|
|
],
|
|
)
|
|
mocked_echo.assert_called_once()
|
|
warning_text = mocked_echo.call_args.args[0]
|
|
self.assertIn("warning: summary count mismatch", warning_text)
|
|
|
|
def test_flatten_costco_data_preserves_discount_rows(self):
|
|
summary_payload = {
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "abc",
|
|
"tenderArray": [{"tenderDescription": "VISA"}],
|
|
"couponArray": [{"upcnumberCoupon": "2100003746641"}],
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
detail_payloads = [
|
|
{
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "abc",
|
|
"transactionDate": "2026-03-12",
|
|
"receiptType": "In-Warehouse",
|
|
"total": 10.0,
|
|
"totalItemCount": 2,
|
|
"instantSavings": 5.0,
|
|
"warehouseName": "MT VERNON",
|
|
"warehouseNumber": 1115,
|
|
"warehouseAddress1": "7940 RICHMOND HWY",
|
|
"warehouseCity": "ALEXANDRIA",
|
|
"warehouseState": "VA",
|
|
"warehousePostalCode": "22306",
|
|
"itemArray": [
|
|
{
|
|
"itemNumber": "4873222",
|
|
"itemDescription01": "ALL F&C",
|
|
"itemDescription02": "200OZ 160LOADS P104",
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": 1,
|
|
"itemIdentifier": "E",
|
|
"amount": 19.99,
|
|
"itemUnitPriceAmount": 19.99,
|
|
},
|
|
{
|
|
"itemNumber": "374664",
|
|
"itemDescription01": "/ 4873222",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": -1,
|
|
"itemIdentifier": None,
|
|
"amount": -5,
|
|
"itemUnitPriceAmount": 0,
|
|
},
|
|
],
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
|
|
orders, items = scrape_costco.flatten_costco_data(
|
|
summary_payload, detail_payloads, Path("costco_output/raw")
|
|
)
|
|
|
|
self.assertEqual(1, len(orders))
|
|
self.assertEqual(2, len(items))
|
|
self.assertEqual("false", items[0]["is_discount_line"])
|
|
self.assertEqual("true", items[1]["is_discount_line"])
|
|
self.assertEqual("true", items[1]["is_coupon_line"])
|
|
|
|
def test_flatten_costco_data_uses_composite_summary_lookup_key(self):
|
|
summary_payload = {
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "dup",
|
|
"transactionDateTime": "2026-03-12T16:16:00",
|
|
"tenderArray": [{"tenderDescription": "VISA"}],
|
|
"couponArray": [{"upcnumberCoupon": "111"}],
|
|
},
|
|
{
|
|
"transactionBarcode": "dup",
|
|
"transactionDateTime": "2026-02-14T16:25:00",
|
|
"tenderArray": [{"tenderDescription": "MASTERCARD"}],
|
|
"couponArray": [],
|
|
},
|
|
]
|
|
}
|
|
}
|
|
}
|
|
detail_payloads = [
|
|
{
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "dup",
|
|
"transactionDateTime": "2026-03-12T16:16:00",
|
|
"transactionDate": "2026-03-12",
|
|
"receiptType": "In-Warehouse",
|
|
"total": 10.0,
|
|
"totalItemCount": 1,
|
|
"instantSavings": 5.0,
|
|
"warehouseName": "MT VERNON",
|
|
"warehouseNumber": 1115,
|
|
"warehouseAddress1": "7940 RICHMOND HWY",
|
|
"warehouseCity": "ALEXANDRIA",
|
|
"warehouseState": "VA",
|
|
"warehousePostalCode": "22306",
|
|
"itemArray": [
|
|
{
|
|
"itemNumber": "111",
|
|
"itemDescription01": "/ 111",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": -1,
|
|
"itemIdentifier": None,
|
|
"amount": -5,
|
|
"itemUnitPriceAmount": 0,
|
|
}
|
|
],
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
|
|
orders, items = scrape_costco.flatten_costco_data(
|
|
summary_payload, detail_payloads, Path("costco_output/raw")
|
|
)
|
|
|
|
self.assertEqual("VISA", orders[0]["payment_method"])
|
|
self.assertEqual("true", items[0]["is_coupon_line"])
|
|
self.assertIn("dup-2026-03-12T16-16-00.json", items[0]["raw_order_path"])
|
|
|
|
def test_costco_enricher_parses_size_pack_and_discount(self):
|
|
row = enrich_costco.parse_costco_item(
|
|
order_id="abc",
|
|
order_date="2026-03-12",
|
|
raw_path=Path("costco_output/raw/abc.json"),
|
|
line_no=1,
|
|
item={
|
|
"itemNumber": "60357",
|
|
"itemDescription01": "MIXED PEPPER",
|
|
"itemDescription02": "6-PACK",
|
|
"itemDepartmentNumber": 65,
|
|
"transDepartmentNumber": 65,
|
|
"unit": 1,
|
|
"itemIdentifier": "E",
|
|
"amount": 7.49,
|
|
"itemUnitPriceAmount": 7.49,
|
|
},
|
|
)
|
|
self.assertEqual("60357", row["retailer_item_id"])
|
|
self.assertEqual("MIXED PEPPER", row["item_name_norm"])
|
|
self.assertEqual("6", row["pack_qty"])
|
|
self.assertEqual("count", row["measure_type"])
|
|
self.assertEqual("costco:abc:1", row["normalized_row_id"])
|
|
self.assertEqual("exact_retailer_item_id", row["normalization_basis"])
|
|
self.assertTrue(row["normalized_item_id"])
|
|
self.assertEqual("6", row["normalized_quantity"])
|
|
self.assertEqual("count", row["normalized_quantity_unit"])
|
|
|
|
volume_row = enrich_costco.parse_costco_item(
|
|
order_id="abc",
|
|
order_date="2026-03-12",
|
|
raw_path=Path("costco_output/raw/abc.json"),
|
|
line_no=3,
|
|
item={
|
|
"itemNumber": "1185912",
|
|
"itemDescription01": "KS ALMND BAR US 1.74QTS CN",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 18,
|
|
"transDepartmentNumber": 18,
|
|
"unit": 2,
|
|
"itemIdentifier": "E",
|
|
"amount": 21.98,
|
|
"itemUnitPriceAmount": 10.99,
|
|
},
|
|
)
|
|
self.assertEqual("3.48", volume_row["normalized_quantity"])
|
|
self.assertEqual("qt", volume_row["normalized_quantity_unit"])
|
|
|
|
discount = enrich_costco.parse_costco_item(
|
|
order_id="abc",
|
|
order_date="2026-03-12",
|
|
raw_path=Path("costco_output/raw/abc.json"),
|
|
line_no=2,
|
|
item={
|
|
"itemNumber": "374664",
|
|
"itemDescription01": "/ 4873222",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": -1,
|
|
"itemIdentifier": None,
|
|
"amount": -5,
|
|
"itemUnitPriceAmount": 0,
|
|
},
|
|
)
|
|
self.assertEqual("true", discount["is_discount_line"])
|
|
self.assertEqual("true", discount["is_coupon_line"])
|
|
self.assertEqual("false", discount["is_item"])
|
|
|
|
def test_costco_name_cleanup_removes_dual_weight_and_logistics_artifacts(self):
|
|
mixed_units = enrich_costco.parse_costco_item(
|
|
order_id="abc",
|
|
order_date="2026-03-12",
|
|
raw_path=Path("costco_output/raw/abc.json"),
|
|
line_no=1,
|
|
item={
|
|
"itemNumber": "18600",
|
|
"itemDescription01": "MANDARINS 2.27 KG / 5 LBS",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 65,
|
|
"transDepartmentNumber": 65,
|
|
"unit": 1,
|
|
"itemIdentifier": "E",
|
|
"amount": 7.49,
|
|
"itemUnitPriceAmount": 7.49,
|
|
},
|
|
)
|
|
self.assertEqual("MANDARIN", mixed_units["item_name_norm"])
|
|
self.assertEqual("5", mixed_units["size_value"])
|
|
self.assertEqual("lb", mixed_units["size_unit"])
|
|
|
|
logistics = enrich_costco.parse_costco_item(
|
|
order_id="abc",
|
|
order_date="2026-03-12",
|
|
raw_path=Path("costco_output/raw/abc.json"),
|
|
line_no=2,
|
|
item={
|
|
"itemNumber": "1375005",
|
|
"itemDescription01": "LIFE 6'TABLE MDL #80873U - T12/H3/P36",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 18,
|
|
"transDepartmentNumber": 18,
|
|
"unit": 1,
|
|
"itemIdentifier": "E",
|
|
"amount": 119.98,
|
|
"itemUnitPriceAmount": 119.98,
|
|
},
|
|
)
|
|
self.assertEqual("LIFE 6'TABLE MDL", logistics["item_name_norm"])
|
|
|
|
def test_costco_hash_weight_parses_into_weight_basis(self):
|
|
row = enrich_costco.parse_costco_item(
|
|
order_id="abc",
|
|
order_date="2024-11-29",
|
|
raw_path=Path("costco_output/raw/abc.json"),
|
|
line_no=4,
|
|
item={
|
|
"itemNumber": "999",
|
|
"itemDescription01": "25# FLOUR ALL-PURPOSE HARV P98/100",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": 1,
|
|
"itemIdentifier": "E",
|
|
"amount": 8.79,
|
|
"itemUnitPriceAmount": 8.79,
|
|
},
|
|
)
|
|
self.assertEqual("FLOUR ALL-PURPOSE HARV", row["item_name_norm"])
|
|
self.assertEqual("25", row["size_value"])
|
|
self.assertEqual("lb", row["size_unit"])
|
|
self.assertEqual("weight", row["measure_type"])
|
|
self.assertEqual("25", row["normalized_quantity"])
|
|
self.assertEqual("lb", row["normalized_quantity_unit"])
|
|
self.assertEqual("0.3516", row["price_per_lb"])
|
|
|
|
def test_build_items_enriched_matches_discount_to_item(self):
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
raw_dir = Path(tmpdir) / "raw"
|
|
raw_dir.mkdir()
|
|
payload = {
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "abc",
|
|
"transactionDate": "2026-03-12",
|
|
"itemArray": [
|
|
{
|
|
"itemNumber": "4873222",
|
|
"itemDescription01": "ALL F&C",
|
|
"itemDescription02": "200OZ 160LOADS P104",
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": 1,
|
|
"itemIdentifier": "E",
|
|
"amount": 19.99,
|
|
"itemUnitPriceAmount": 19.99,
|
|
},
|
|
{
|
|
"itemNumber": "374664",
|
|
"itemDescription01": "/ 4873222",
|
|
"itemDescription02": None,
|
|
"itemDepartmentNumber": 14,
|
|
"transDepartmentNumber": 14,
|
|
"unit": -1,
|
|
"itemIdentifier": None,
|
|
"amount": -5,
|
|
"itemUnitPriceAmount": 0,
|
|
},
|
|
],
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
(raw_dir / "abc.json").write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
rows = enrich_costco.build_items_enriched(raw_dir)
|
|
|
|
purchase_row = next(row for row in rows if row["is_discount_line"] == "false")
|
|
discount_row = next(row for row in rows if row["is_discount_line"] == "true")
|
|
self.assertEqual("-5", purchase_row["matched_discount_amount"])
|
|
self.assertEqual("14.99", purchase_row["net_line_total"])
|
|
self.assertIn("matched_discount=4873222", purchase_row["parse_notes"])
|
|
self.assertIn("matched_to_item=4873222", discount_row["parse_notes"])
|
|
|
|
def test_main_writes_summary_request_metadata(self):
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
outdir = Path(tmpdir) / "costco_output"
|
|
summary_payload = {
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"inWarehouse": 1,
|
|
"gasStation": 0,
|
|
"carWash": 0,
|
|
"gasAndCarWash": 0,
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "abc",
|
|
"receiptType": "In-Warehouse",
|
|
"tenderArray": [],
|
|
"couponArray": [],
|
|
}
|
|
],
|
|
}
|
|
}
|
|
}
|
|
detail_payload = {
|
|
"data": {
|
|
"receiptsWithCounts": {
|
|
"receipts": [
|
|
{
|
|
"transactionBarcode": "abc",
|
|
"transactionDate": "2026-03-12",
|
|
"receiptType": "In-Warehouse",
|
|
"total": 10.0,
|
|
"totalItemCount": 1,
|
|
"instantSavings": 0,
|
|
"warehouseName": "MT VERNON",
|
|
"warehouseNumber": 1115,
|
|
"warehouseAddress1": "7940 RICHMOND HWY",
|
|
"warehouseCity": "ALEXANDRIA",
|
|
"warehouseState": "VA",
|
|
"warehousePostalCode": "22306",
|
|
"itemArray": [],
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
metadata = [
|
|
{
|
|
"startDate": "1/01/2026",
|
|
"endDate": "3/31/2026",
|
|
"text": "custom",
|
|
"documentType": "all",
|
|
"documentSubType": "all",
|
|
"returnedReceipts": 1,
|
|
"returnedInWarehouseReceipts": 1,
|
|
"inWarehouse": 1,
|
|
"gasStation": 0,
|
|
"carWash": 0,
|
|
"gasAndCarWash": 0,
|
|
"countMismatch": False,
|
|
}
|
|
]
|
|
|
|
with mock.patch.object(
|
|
scrape_costco,
|
|
"load_config",
|
|
return_value={
|
|
"authorization": "",
|
|
"client_id": "4900eb1f-0c10-4bd9-99c3-c59e6c1ecebf",
|
|
"client_identifier": "481b1aec-aa3b-454b-b81b-48187e28f205",
|
|
},
|
|
), mock.patch.object(
|
|
scrape_costco,
|
|
"find_firefox_profile_dir",
|
|
return_value=Path("/tmp/profile"),
|
|
), mock.patch.object(
|
|
scrape_costco,
|
|
"load_costco_browser_headers",
|
|
return_value={
|
|
"costco-x-authorization": "Bearer header.payload.signature",
|
|
"costco-x-wcs-clientId": "4900eb1f-0c10-4bd9-99c3-c59e6c1ecebf",
|
|
"client-identifier": "481b1aec-aa3b-454b-b81b-48187e28f205",
|
|
},
|
|
), mock.patch.object(
|
|
scrape_costco, "build_session", return_value=object()
|
|
), mock.patch.object(
|
|
scrape_costco,
|
|
"fetch_summary_windows",
|
|
return_value=(summary_payload, metadata),
|
|
), mock.patch.object(
|
|
scrape_costco,
|
|
"graphql_post",
|
|
return_value=detail_payload,
|
|
):
|
|
scrape_costco.main.callback(
|
|
outdir=str(outdir),
|
|
document_type="all",
|
|
document_sub_type="all",
|
|
window_days=92,
|
|
months_back=3,
|
|
firefox_profile_dir=None,
|
|
)
|
|
|
|
metadata_path = outdir / "raw" / "summary_requests.json"
|
|
self.assertTrue(metadata_path.exists())
|
|
saved_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(metadata, saved_metadata)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|