15 Commits
Author SHA1 Message Date
C dcccdcd9ec fix pricing not being entered 2026-01-07 11:36:24 -08:00
C dbc1115249 fix hang 2026-01-07 11:25:45 -08:00
C dbe5cd6146 final commit before first prod test 2026-01-07 09:59:36 -08:00
C a6e2397046 fix prod hang 2026-01-05 16:41:25 -08:00
C 2da61539cc fix worker warning 2026-01-05 16:32:37 -08:00
C bf65965b26 fix banner 2026-01-05 16:26:19 -08:00
C bcf625d2b0 add stats, versioning 2026-01-05 16:23:44 -08:00
C dfd2c2f07d add caching, add to stats 2026-01-05 16:01:36 -08:00
C ab3297acfb fix env thing 2026-01-05 15:44:04 -08:00
C 90aca6cca4 hopefully fix workers not kloading the key 2026-01-05 15:42:54 -08:00
C 199995b947 fix key please 2026-01-05 15:37:50 -08:00
C 2f7769e2c3 imporerror 2026-01-05 15:27:34 -08:00
C 7ca181bdd6 Merge remote-tracking branch 'origin/rewrite' into rewrite 2026-01-05 15:24:55 -08:00
C 1e96dead07 fix imports and price 2026-01-05 15:24:50 -08:00
C 248950adb9 fix imports 2026-01-05 15:23:33 -08:00
11 changed files with 247 additions and 145 deletions

No files matched your search

+10 -3
View File
@@ -6,17 +6,24 @@ BRICKSET_API_KEY = "your_brickset_api_key"
# Square environment - "sandbox" for testing, "production" for live store # Square environment - "sandbox" for testing, "production" for live store
SQUARE_ENV = "sandbox" # or "production" SQUARE_ENV = "sandbox" # or "production"
SQUARE_WEBHOOK_URL = "https://yourdomain.com/hook" # Public webhook URL that Square will POST events to
SQUARE_WEBHOOK_URL = "https://yourdomain.com/webhook"
# Sandbox credentials - test environment (fake data, safe for development) # Sandbox credentials - test environment (fake data, safe for development)
SANDBOX_SQUARE_APP_ID = "your_sandbox_app_id" SANDBOX_SQUARE_APP_ID = "your_sandbox_app_id"
SANDBOX_SQUARE_TOKEN = "your_sandbox_access_token" SANDBOX_SQUARE_TOKEN = "your_sandbox_access_token"
SANDBOX_SQUARE_WEBHOOK_INVENTORY_ID = "your_sandbox_webhook_signature_key"
# Sandbox webhook subscription ID and signature key for verifying webhook requests
SANDBOX_SQUARE_WEBHOOK_SUB_ID = "your_sandbox_webhook_subscription_id"
SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY = "your_sandbox_webhook_signature_key"
# Production credentials - live store (real inventory and sales) # Production credentials - live store (real inventory and sales)
PROD_SQUARE_APP_ID = "your_production_app_id" PROD_SQUARE_APP_ID = "your_production_app_id"
PROD_SQUARE_TOKEN = "your_production_access_token" PROD_SQUARE_TOKEN = "your_production_access_token"
PROD_SQUARE_WEBHOOK_INVENTORY_ID = "your_production_webhook_signature_key"
# Production webhook subscription ID and signature key for verifying webhook requests
PROD_SQUARE_WEBHOOK_SUB_ID = "your_production_webhook_subscription_id"
PROD_SQUARE_WEBHOOK_SIGNATURE_KEY = "your_production_webhook_signature_key"
# Category ID for LEGO items in Square catalog (find in Dashboard → Items → Categories) # Category ID for LEGO items in Square catalog (find in Dashboard → Items → Categories)
SQUARE_LEGO_CAT_ID = "your_lego_category_id" SQUARE_LEGO_CAT_ID = "your_lego_category_id"
+1
View File
@@ -0,0 +1 @@
0.3.0
+26 -19
View File
@@ -1,8 +1,18 @@
import multiprocessing import multiprocessing
import os import os
import git
import src.banner as banner import src.banner as banner
def get_git_commit():
"""Get first 6 characters of current git commit hash"""
try:
repo = git.Repo(search_parent_directories=True)
return repo.head.commit.hexsha[:6]
except Exception:
return "unknown"
# Server socket # Server socket
bind = "127.0.0.1:7433" # nginx will proxy to this bind = "127.0.0.1:7433" # nginx will proxy to this
backlog = 2048 backlog = 2048
@@ -54,26 +64,13 @@ def on_starting(server):
"""Called just before the master process is initialized.""" """Called just before the master process is initialized."""
print("Studspider WEB starting...") print("Studspider WEB starting...")
# Load .env file before workers start # Verify .env file exists (workers will load it via get_sq_env.py)
import dotenv
from pathlib import Path from pathlib import Path
env_path = Path(__file__).parent.parent / ".env" env_path = Path(__file__).parent.parent / ".env"
print(f"[@DEBUG] Looking for .env at: {env_path.absolute()}") print(f"[@DEBUG] Looking for .env at: {env_path.absolute()}")
if env_path.exists(): if env_path.exists():
dotenv.load_dotenv(env_path) print(f"✓ .env file found (workers will load via get_sq_env.py)")
print(f"✓ Loaded environment from {env_path}")
# Verify key was loaded
import os
key = os.getenv("PROD_SQUARE_WEBHOOK_SIGNATURE_KEY") or os.getenv(
"SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY"
)
if key:
print(f"✓ SQUARE_WEBHOOK_SIGNATURE_KEY loaded: {key[:10]}...")
else:
print(f"⚠ SQUARE_WEBHOOK_SIGNATURE_KEY not found in .env")
else: else:
print(f"⚠ No .env file found at {env_path.absolute()}") print(f"⚠ No .env file found at {env_path.absolute()}")
@@ -84,11 +81,21 @@ def on_reload(server):
def when_ready(server): def when_ready(server):
print(banner.generate("studspider v0.1")) commit_hash = get_git_commit()
banner_text = banner.generate("studspider v0.3.0")
# Add commit hash to the last line of banner
banner_lines = banner_text.rstrip("\n").split("\n")
if banner_lines:
banner_lines[-1] += f" - {commit_hash}"
print("\n".join(banner_lines))
print("Built for PLAY-WELL TEKNOLOGIES by contact@studspider.sh\n\n") print("Built for PLAY-WELL TEKNOLOGIES by contact@studspider.sh\n\n")
print(f"STUDSPIDER HOOK v0.1 RUNNING ON {bind}") print("PRODUCTION MODE\n" if os.getenv("SQUARE_ENV") == "production" else "SANDBOX MODE\n")
print(f"WORKERS: {workers}") print(f"HOST:\t{bind}")
print(f"WORKING CLASS: {worker_class}\n") print(f"WORKERS:\t{workers}")
print(f"WORKER TYPE:\t{worker_class}")
print(f"{'=' * 60}\n")
def worker_int(worker): def worker_int(worker):
+3 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "studpider" name = "studpider"
version = "0.1.0" version = "0.3.0"
description = "" description = ""
authors = [{ name = "CF", email = "ph0sgene@pm.me" }] authors = [{ name = "CF", email = "ph0sgene@pm.me" }]
requires-python = ">=3.13,<3.14" requires-python = ">=3.13,<3.14"
@@ -21,6 +21,8 @@ dependencies = [
"fastapi>=0.128.0", "fastapi>=0.128.0",
"uvicorn>=0.40.0", "uvicorn>=0.40.0",
"gunicorn>=23.0.0", "gunicorn>=23.0.0",
"async-lru>=2.0.4",
"gitpython>=3.1.0",
] ]
[build-system] [build-system]
+1 -1
View File
@@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="studspider", name="studspider",
version="0.1", version="0.3.0",
description="Bricket + Square integration application", description="Bricket + Square integration application",
author="CF", author="CF",
author_email="chris-forst@pm.me", author_email="chris-forst@pm.me",
+15 -11
View File
@@ -7,10 +7,13 @@ import asyncio
import os import os
import aiohttp import aiohttp
import json import json
import math
import dotenv import dotenv
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from datetime import datetime, timedelta from datetime import datetime, timedelta
from async_lru import alru_cache
from . import throttler from . import throttler
from . import get_sq_env
Throttler = throttler.GlobalThrottler Throttler = throttler.GlobalThrottler
@@ -20,11 +23,12 @@ dotenv.load_dotenv()
class AsyncBricksetAPI: class AsyncBricksetAPI:
"""Asynchronous client for the Brickset API""" """Asynchronous client for the Brickset API"""
def __init__(self, debug: bool = False, logger=None): def __init__(self, debug: bool = None, logger=None):
self.username = os.getenv("BRICKSET_USERNAME") self.username = os.getenv("BRICKSET_USERNAME")
self.password = os.getenv("BRICKSET_PASS") self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY") self.api_key = os.getenv("BRICKSET_API_KEY")
self.debug = debug # Use DEBUG_MODE from env if debug not explicitly set
self.debug = debug if debug is not None else get_sq_env.DEBUG_MODE
self.headers = {"UserAgent": "studspider/0.1"} self.headers = {"UserAgent": "studspider/0.1"}
self.throttler = Throttler(rate_limit=500, period=60) self.throttler = Throttler(rate_limit=500, period=60)
self.user_hash = None self.user_hash = None
@@ -154,9 +158,10 @@ class AsyncBricksetAPI:
self.logger.info("Successfully authenticated with Brickset API") self.logger.info("Successfully authenticated with Brickset API")
return result return result
@alru_cache(maxsize=1000, ttl=604800) # Cache 1000 sets, 7 days TTL
async def get_set(self, set_number: int) -> Dict[str, Any]: async def get_set(self, set_number: int) -> Dict[str, Any]:
""" """
Fetch LEGO set information asynchronously Fetch LEGO set information asynchronously (cached for 7 days)
Args: Args:
set_number (str): LEGO set number set_number (str): LEGO set number
@@ -215,22 +220,21 @@ class AsyncBricksetAPI:
barcode_data = set_data.get("barcode", {}) barcode_data = set_data.get("barcode", {})
upc_code = barcode_data.get("UPC") or barcode_data.get("EAN") upc_code = barcode_data.get("UPC") or barcode_data.get("EAN")
# Get retail price before rounding
retail_price = set_data["LEGOCom"]["US"]["retailPrice"]
our_set = { our_set = {
"set_num": set_data["number"], "set_num": set_data["number"],
"set_name": set_data["name"], "set_name": set_data["name"],
"set_theme": set_data["theme"], "set_theme": set_data["theme"],
"set_image": set_data["image"]["thumbnailURL"], "set_image": set_data["image"]["thumbnailURL"],
"set_barcode": upc_code, "set_barcode": upc_code,
"set_retailprice": round( "set_retailprice": round(retail_price),
set_data["LEGOCom"]["US"]["retailPrice"]
),
} }
# Calculate adjusted price (10% markup) # Calculate adjusted price (10% markup), then round up
retail_price = our_set["set_retailprice"] adjusted_price = retail_price * 1.10
our_set["set_adjustredprice"] = retail_price + ( our_set["set_adjustredprice"] = math.ceil(adjusted_price)
(10 / 100) * retail_price
)
# Set PW inventory name with theme # Set PW inventory name with theme
our_set["pw_name"] = ( our_set["pw_name"] = (
+29 -14
View File
@@ -4,6 +4,7 @@ import json
import asyncio import asyncio
import uuid import uuid
import re import re
import traceback
from time import time from time import time
import aiohttp import aiohttp
from io import BytesIO from io import BytesIO
@@ -123,8 +124,9 @@ async def get_item_by_variation_id(variation_id: str) -> dict:
cursor = None cursor = None
page_count = 0 page_count = 0
max_pages = 50 # Safety limit to prevent infinite loops
while True: while page_count < max_pages:
page_count += 1 page_count += 1
print(f"[@DEBUG] Searching page {page_count}...") print(f"[@DEBUG] Searching page {page_count}...")
@@ -150,6 +152,7 @@ async def get_item_by_variation_id(variation_id: str) -> dict:
raise ValueError(f"No items found in LEGO category") raise ValueError(f"No items found in LEGO category")
else: else:
# No more results # No more results
print(f"[@DEBUG] No more results on page {page_count}")
break break
# Search through items on this page to find which has this variation # Search through items on this page to find which has this variation
@@ -187,8 +190,15 @@ async def get_item_by_variation_id(variation_id: str) -> dict:
cursor = result_json.get("cursor") cursor = result_json.get("cursor")
if not cursor: if not cursor:
# No more pages # No more pages
print(f"[@DEBUG] No more pages after page {page_count}")
break break
# If we get here, variation wasn't found
if page_count >= max_pages:
raise ValueError(
f"Variation {variation_id} not found after searching {max_pages} pages (possible infinite loop prevented)"
)
raise ValueError( raise ValueError(
f"Variation {variation_id} not found in any LEGO category item (searched {page_count} pages)" f"Variation {variation_id} not found in any LEGO category item (searched {page_count} pages)"
) )
@@ -293,7 +303,6 @@ async def update_itm(item_id: str, lego_set: dict, item_data: dict = None):
pw_name = lego_set["pw_name"] pw_name = lego_set["pw_name"]
set_adjustedprice = lego_set["set_adjustredprice"] set_adjustedprice = lego_set["set_adjustredprice"]
# Convert price to cents for Square API
price_in_cents = int(set_adjustedprice * 100) price_in_cents = int(set_adjustedprice * 100)
# Retry logic for version conflicts (race conditions with concurrent updates) # Retry logic for version conflicts (race conditions with concurrent updates)
@@ -376,11 +385,19 @@ async def update_itm(item_id: str, lego_set: dict, item_data: dict = None):
"item_variation_data", {} "item_variation_data", {}
).copy() ).copy()
# Update only specific fields # Check if this variation has VARIABLE_PRICING
updated_var_data["price_money"] = { current_pricing_type = updated_var_data.get("pricing_type")
"amount": price_in_cents,
"currency": "USD", # Only set price if not using variable pricing
} if current_pricing_type != "VARIABLE_PRICING":
updated_var_data["price_money"] = {
"amount": price_in_cents,
"currency": "USD",
}
print(f"[@DEBUG] Setting price: ${set_adjustedprice} for variation {variation_id}")
else:
print(f"[@WARNING] Skipping price update for {variation_id} - has VARIABLE_PRICING enabled")
updated_var_data["sku"] = set_number # LEGO set number updated_var_data["sku"] = set_number # LEGO set number
updated_var_data["upc"] = ( updated_var_data["upc"] = (
set_upc if set_upc else None set_upc if set_upc else None
@@ -395,7 +412,11 @@ async def update_itm(item_id: str, lego_set: dict, item_data: dict = None):
updated_variations.append(updated_var) updated_variations.append(updated_var)
variation_found = True variation_found = True
else: else:
# Keep other variations as-is # Keeprice_money": {
"amount": price_in_cents,
"currency": "USD",
},
"p other variations as-is
updated_variations.append(existing_var) updated_variations.append(existing_var)
# If variation wasn't found (shouldn't happen), create new one # If variation wasn't found (shouldn't happen), create new one
@@ -407,10 +428,6 @@ async def update_itm(item_id: str, lego_set: dict, item_data: dict = None):
"item_id": item_id, "item_id": item_id,
"name": "Regular", "name": "Regular",
"pricing_type": "FIXED_PRICING", "pricing_type": "FIXED_PRICING",
"price_money": {
"amount": price_in_cents,
"currency": "USD",
},
"sku": set_number, "sku": set_number,
"upc": set_upc if set_upc else None, "upc": set_upc if set_upc else None,
}, },
@@ -492,8 +509,6 @@ async def update_itm(item_id: str, lego_set: dict, item_data: dict = None):
except Exception as e: except Exception as e:
print(f"[@ERROR] Failed to update item {item_id}: {e}") print(f"[@ERROR] Failed to update item {item_id}: {e}")
import traceback
traceback.print_exc() traceback.print_exc()
return False return False
+10 -3
View File
@@ -1,14 +1,21 @@
import os import os
import dotenv import dotenv
from pathlib import Path
from square.environment import SquareEnvironment from square.environment import SquareEnvironment
dotenv.load_dotenv() # Load .env from project root (parent of src/)
env_path = Path(__file__).parent.parent / ".env"
dotenv.load_dotenv(env_path)
LEGO_CAT_ID = os.getenv("SQUARE_LEGO_CAT_ID") LEGO_CAT_ID = os.getenv("SQUARE_LEGO_CAT_ID")
SQUARE_WEBHOOK_URL = os.getenv("SQUARE_WEBHOOK_URL") SQUARE_WEBHOOK_URL = os.getenv("SQUARE_WEBHOOK_URL")
if os.getenv("SQUARE_ENV") == "production": # Check environment mode
IS_PRODUCTION = os.getenv("SQUARE_ENV") == "production"
DEBUG_MODE = not IS_PRODUCTION # Debug logging enabled in sandbox/dev
if IS_PRODUCTION:
SQUARE_APP_ID = os.getenv("PROD_SQUARE_APP_ID") SQUARE_APP_ID = os.getenv("PROD_SQUARE_APP_ID")
SQUARE_TOKEN = os.getenv("PROD_SQUARE_TOKEN") SQUARE_TOKEN = os.getenv("PROD_SQUARE_TOKEN")
SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("PROD_SQUARE_WEBHOOK_SIGNATURE_KEY") SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("PROD_SQUARE_WEBHOOK_SIGNATURE_KEY")
@@ -19,4 +26,4 @@ else:
SQUARE_TOKEN = os.getenv("SANDBOX_SQUARE_TOKEN") SQUARE_TOKEN = os.getenv("SANDBOX_SQUARE_TOKEN")
SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY") SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY")
SQUARE_WEBHOOK_SUB_ID = os.getenv("SANDBOX_SQUARE_WEBHOOK_SUB_ID") SQUARE_WEBHOOK_SUB_ID = os.getenv("SANDBOX_SQUARE_WEBHOOK_SUB_ID")
environment = SquareEnvironment.SANDBOX environment = SquareEnvironment.SANDBOX
+31 -25
View File
@@ -96,7 +96,8 @@ async def enrich_single_item(item_id: str, item_data: dict = None, logger=None)
print(f"[@DEBUG] Extracted set number: {set_number}") print(f"[@DEBUG] Extracted set number: {set_number}")
# Fetch Brickset data (with throttling) # Fetch Brickset data (with throttling)
brickset_client = abricksetapi.AsyncBricksetAPI(debug=True, logger=logger) # debug flag will be auto-set based on SQUARE_ENV
brickset_client = abricksetapi.AsyncBricksetAPI(logger=logger)
lego_set = await brickset_client.throttled_get_set(set_number) lego_set = await brickset_client.throttled_get_set(set_number)
if logger: if logger:
@@ -104,24 +105,10 @@ async def enrich_single_item(item_id: str, item_data: dict = None, logger=None)
else: else:
print(f"[@DEBUG] Retrieved Brickset data for {lego_set['pw_name']}") print(f"[@DEBUG] Retrieved Brickset data for {lego_set['pw_name']}")
# Check if manual price review needed # Note: Price updates disabled - enriching without price
if lego_set.get("requires_manual_price"): # REVIEW table logic skipped
if logger:
logger.info(f"Item {item_id} requires manual price review (≥$110)")
else:
print(f"[@DEBUG] Item {item_id} requires manual price review")
await db.upsert( # Update Square item with enriched data (without price)
"REVIEW",
{
"sq_object": item_id,
"price": lego_set["set_adjustredprice"],
"date_modified": int(time()),
},
"sq_object",
)
# Update Square item with enriched data
success = await asquare.update_itm(item_id, lego_set, item_data=item) success = await asquare.update_itm(item_id, lego_set, item_data=item)
if success: if success:
@@ -179,18 +166,37 @@ async def enrich_bulk(logger=None) -> None:
else: else:
print(f"[@DEBUG] Found {len(items)} items in LEGO category") print(f"[@DEBUG] Found {len(items)} items in LEGO category")
# Enrich each item # Enrich each item with concurrent batches
successful = 0 successful = 0
failed = 0 failed = 0
batch_size = 20 # Process 20 items concurrently
for item in items: for i in range(0, len(items), batch_size):
item_id = item["id"] batch = items[i : i + batch_size]
result = await enrich_single_item(item_id, logger=logger) batch_num = (i // batch_size) + 1
total_batches = (len(items) + batch_size - 1) // batch_size
if result: if logger:
successful += 1 logger.info(
f"Processing batch {batch_num}/{total_batches} ({len(batch)} items)"
)
else: else:
failed += 1 print(
f"[@DEBUG] Processing batch {batch_num}/{total_batches} ({len(batch)} items)"
)
# Process batch concurrently
tasks = [enrich_single_item(item["id"], logger=logger) for item in batch]
results = await asyncio.gather(*tasks, return_exceptions=True)
# Count results
for result in results:
if isinstance(result, Exception):
failed += 1
elif result:
successful += 1
else:
failed += 1
if logger: if logger:
logger.info( logger.info(
+116 -55
View File
@@ -4,15 +4,24 @@ Queries Square catalog to collect metrics about LEGO inventory quality.
""" """
import json import json
import git
from typing import Dict, Any from typing import Dict, Any
from square import AsyncSquare from square import AsyncSquare
import get_sq_env from . import get_sq_env
from . import abricksetapi
def get_git_commit() -> str:
"""Get first 6 characters of current git commit hash"""
try:
repo = git.Repo(search_parent_directories=True)
return repo.head.commit.hexsha[:6]
except Exception:
return "unknown"
# Initialize Square client # Initialize Square client
client = AsyncSquare( client = AsyncSquare(token=get_sq_env.SQUARE_TOKEN, environment=get_sq_env.environment)
token=get_sq_env.SQUARE_TOKEN,
environment=get_sq_env.environment
)
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
@@ -20,7 +29,7 @@ LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
async def get_daily_stats() -> Dict[str, Any]: async def get_daily_stats() -> Dict[str, Any]:
""" """
Gather all metrics for daily_stats table in a single pass. Gather all metrics for daily_stats table in a single pass.
Returns: Returns:
dict: { dict: {
"total_items": int, "total_items": int,
@@ -40,38 +49,38 @@ async def get_daily_stats() -> Dict[str, Any]:
} }
}, },
) )
result_json = json.loads(result.model_dump_json()) result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", []) objects = result_json.get("objects", [])
# Initialize counters # Initialize counters
total_items = 0 total_items = 0
enriched_items = 0 enriched_items = 0
missing_upcs = 0 missing_upcs = 0
missing_images = 0 missing_images = 0
total_catalog_value = 0.0 total_catalog_value = 0.0
for obj in objects: for obj in objects:
total_items += 1 total_items += 1
item_data = obj.get("item_data", {}) item_data = obj.get("item_data", {})
# Check if item has all enrichment data # Check if item has all enrichment data
has_upc = False has_upc = False
has_image = False has_image = False
has_proper_name = False has_proper_name = False
item_price = 0.0 item_price = 0.0
# Check variations for UPC and price # Check variations for UPC and price
variations = item_data.get("variations", []) variations = item_data.get("variations", [])
if variations: if variations:
var_data = variations[0].get("item_variation_data", {}) var_data = variations[0].get("item_variation_data", {})
# Check UPC # Check UPC
if var_data.get("upc"): if var_data.get("upc"):
has_upc = True has_upc = True
else: else:
missing_upcs += 1 missing_upcs += 1
# Get price (stored in cents) # Get price (stored in cents)
price_money = var_data.get("price_money", {}) price_money = var_data.get("price_money", {})
if price_money and price_money.get("amount"): if price_money and price_money.get("amount"):
@@ -79,22 +88,22 @@ async def get_daily_stats() -> Dict[str, Any]:
total_catalog_value += item_price total_catalog_value += item_price
else: else:
missing_upcs += 1 missing_upcs += 1
# Check for image # Check for image
if item_data.get("image_ids"): if item_data.get("image_ids"):
has_image = True has_image = True
else: else:
missing_images += 1 missing_images += 1
# Check if name follows format: "12345 - THEME Set Name" # Check if name follows format: "12345 - THEME Set Name"
name = item_data.get("name", "") name = item_data.get("name", "")
if " - " in name and name.split()[0].isdigit() and len(name.split()[0]) == 5: if " - " in name and name.split()[0].isdigit() and len(name.split()[0]) == 5:
has_proper_name = True has_proper_name = True
# Item is "enriched" if it has UPC, image, and proper name format # Item is "enriched" if it has UPC, image, and proper name format
if has_upc and has_image and has_proper_name: if has_upc and has_image and has_proper_name:
enriched_items += 1 enriched_items += 1
return { return {
"total_items": total_items, "total_items": total_items,
"enriched_items": enriched_items, "enriched_items": enriched_items,
@@ -104,12 +113,62 @@ async def get_daily_stats() -> Dict[str, Any]:
} }
async def get_total_items() -> int: async def get_brickset_cache_stats() -> Dict[str, Any]:
""" """
Count total items in LEGO category. Get Brickset API cache statistics.
Returns: Returns:
int: Total number of items dict: {
"cache_enabled": bool,
"cache_hits": int,
"cache_misses": int,
"cache_size": int,
"cache_maxsize": int,
"hit_rate": float (0-100%)
}
"""
# Create a temporary client instance to access cache info
temp_client = abricksetapi.AsyncBricksetAPI()
# Check if get_set has cache_info (it should with @alru_cache)
if hasattr(temp_client.get_set, "cache_info"):
info = await temp_client.get_set.cache_info()
hits = info.hits
misses = info.misses
total = hits + misses
hit_rate = (hits / total * 100) if total > 0 else 0.0
return {
"cache_enabled": True,
"cache_hits": hits,
"cache_misses": misses,
"cache_size": info.currsize,
"cache_maxsize": info.maxsize,
"hit_rate": round(hit_rate, 2),
}
else:
return {
"cache_enabled": False,
"cache_hits": 0,
"cache_misses": 0,
"cache_size": 0,
"cache_maxsize": 0,
"hit_rate": 0.0,
}
async def get_daily_stats() -> Dict[str, Any]:
"""
Gather all metrics for daily_stats table in a single pass.
Returns:
dict: {
"total_items": int,
"enriched_items": int,
"missing_upcs": int,
"missing_images": int,
"total_catalog_value": float
}
""" """
result = await client.catalog.search( result = await client.catalog.search(
object_types=["ITEM"], object_types=["ITEM"],
@@ -120,7 +179,7 @@ async def get_total_items() -> int:
} }
}, },
) )
result_json = json.loads(result.model_dump_json()) result_json = json.loads(result.model_dump_json())
return len(result_json.get("objects", [])) return len(result_json.get("objects", []))
@@ -128,7 +187,7 @@ async def get_total_items() -> int:
async def get_enriched_count() -> int: async def get_enriched_count() -> int:
""" """
Count items with complete enrichment (UPC + image + proper name format). Count items with complete enrichment (UPC + image + proper name format).
Returns: Returns:
int: Number of fully enriched items int: Number of fully enriched items
""" """
@@ -141,25 +200,25 @@ async def get_enriched_count() -> int:
} }
}, },
) )
result_json = json.loads(result.model_dump_json()) result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", []) objects = result_json.get("objects", [])
enriched_count = 0 enriched_count = 0
for obj in objects: for obj in objects:
item_data = obj.get("item_data", {}) item_data = obj.get("item_data", {})
# Check UPC # Check UPC
has_upc = False has_upc = False
variations = item_data.get("variations", []) variations = item_data.get("variations", [])
if variations: if variations:
var_data = variations[0].get("item_variation_data", {}) var_data = variations[0].get("item_variation_data", {})
has_upc = bool(var_data.get("upc")) has_upc = bool(var_data.get("upc"))
# Check image # Check image
has_image = bool(item_data.get("image_ids")) has_image = bool(item_data.get("image_ids"))
# Check name format: "12345 - THEME Set Name" # Check name format: "12345 - THEME Set Name"
name = item_data.get("name", "") name = item_data.get("name", "")
has_proper_name = False has_proper_name = False
@@ -167,17 +226,17 @@ async def get_enriched_count() -> int:
first_part = name.split()[0] first_part = name.split()[0]
if first_part.isdigit() and len(first_part) == 5: if first_part.isdigit() and len(first_part) == 5:
has_proper_name = True has_proper_name = True
if has_upc and has_image and has_proper_name: if has_upc and has_image and has_proper_name:
enriched_count += 1 enriched_count += 1
return enriched_count return enriched_count
async def get_missing_upcs() -> int: async def get_missing_upcs() -> int:
""" """
Count items without UPC barcodes. Count items without UPC barcodes.
Returns: Returns:
int: Number of items missing UPCs int: Number of items missing UPCs
""" """
@@ -190,31 +249,31 @@ async def get_missing_upcs() -> int:
} }
}, },
) )
result_json = json.loads(result.model_dump_json()) result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", []) objects = result_json.get("objects", [])
missing_count = 0 missing_count = 0
for obj in objects: for obj in objects:
item_data = obj.get("item_data", {}) item_data = obj.get("item_data", {})
variations = item_data.get("variations", []) variations = item_data.get("variations", [])
if not variations: if not variations:
missing_count += 1 missing_count += 1
continue continue
var_data = variations[0].get("item_variation_data", {}) var_data = variations[0].get("item_variation_data", {})
if not var_data.get("upc"): if not var_data.get("upc"):
missing_count += 1 missing_count += 1
return missing_count return missing_count
async def get_missing_images() -> int: async def get_missing_images() -> int:
""" """
Count items without images. Count items without images.
Returns: Returns:
int: Number of items missing images int: Number of items missing images
""" """
@@ -227,24 +286,24 @@ async def get_missing_images() -> int:
} }
}, },
) )
result_json = json.loads(result.model_dump_json()) result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", []) objects = result_json.get("objects", [])
missing_count = 0 missing_count = 0
for obj in objects: for obj in objects:
item_data = obj.get("item_data", {}) item_data = obj.get("item_data", {})
if not item_data.get("image_ids"): if not item_data.get("image_ids"):
missing_count += 1 missing_count += 1
return missing_count return missing_count
async def get_total_catalog_value() -> float: async def get_total_catalog_value() -> float:
""" """
Calculate total dollar value of all items in LEGO category. Calculate total dollar value of all items in LEGO category.
Returns: Returns:
float: Total catalog value in dollars float: Total catalog value in dollars
""" """
@@ -257,42 +316,44 @@ async def get_total_catalog_value() -> float:
} }
}, },
) )
result_json = json.loads(result.model_dump_json()) result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", []) objects = result_json.get("objects", [])
total_value = 0.0 total_value = 0.0
for obj in objects: for obj in objects:
item_data = obj.get("item_data", {}) item_data = obj.get("item_data", {})
variations = item_data.get("variations", []) variations = item_data.get("variations", [])
if variations: if variations:
var_data = variations[0].get("item_variation_data", {}) var_data = variations[0].get("item_variation_data", {})
price_money = var_data.get("price_money", {}) price_money = var_data.get("price_money", {})
if price_money and price_money.get("amount"): if price_money and price_money.get("amount"):
# Price is stored in cents, convert to dollars # Price is stored in cents, convert to dollars
total_value += price_money["amount"] / 100.0 total_value += price_money["amount"] / 100.0
return round(total_value, 2) return round(total_value, 2)
# Example usage # Example usage
if __name__ == "__main__": if __name__ == "__main__":
import asyncio import asyncio
async def main(): async def main():
print("Gathering daily statistics for LEGO catalog...") print("Gathering daily statistics for LEGO catalog...")
stats = await get_daily_stats() stats = await get_daily_stats()
print(f"\n=== Daily Stats ===") print(f"\n=== Daily Stats ===")
print(f"Total Items: {stats['total_items']}") print(f"Total Items: {stats['total_items']}")
print(f"Enriched Items: {stats['enriched_items']}") print(f"Enriched Items: {stats['enriched_items']}")
print(f"Missing UPCs: {stats['missing_upcs']}") print(f"Missing UPCs: {stats['missing_upcs']}")
print(f"Missing Images: {stats['missing_images']}") print(f"Missing Images: {stats['missing_images']}")
print(f"Total Catalog Value: ${stats['total_catalog_value']:,.2f}") print(f"Total Catalog Value: ${stats['total_catalog_value']:,.2f}")
print(f"Completion Rate: {(stats['enriched_items'] / stats['total_items'] * 100):.1f}%") print(
f"Completion Rate: {(stats['enriched_items'] / stats['total_items'] * 100):.1f}%"
)
asyncio.run(main()) asyncio.run(main())
+5 -13
View File
@@ -15,18 +15,6 @@ SQUARE_WEBHOOK_SIGNATURE_KEY = sq_env.SQUARE_WEBHOOK_SIGNATURE_KEY
SQUARE_WEBHOOK_URL = sq_env.SQUARE_WEBHOOK_URL SQUARE_WEBHOOK_URL = sq_env.SQUARE_WEBHOOK_URL
SQUARE_LEGO_CAT_ID = sq_env.LEGO_CAT_ID SQUARE_LEGO_CAT_ID = sq_env.LEGO_CAT_ID
# Debug: Print configuration (redact key)
if SQUARE_WEBHOOK_SIGNATURE_KEY:
print(f"[CONFIG] Signature key loaded: {SQUARE_WEBHOOK_SIGNATURE_KEY[:10]}...")
else:
print("[CONFIG] WARNING: SQUARE_WEBHOOK_SIGNATURE_KEY not set!")
if SQUARE_WEBHOOK_URL:
print(f"[CONFIG] Webhook URL: {SQUARE_WEBHOOK_URL}")
else:
print("[CONFIG] WARNING: SQUARE_WEBHOOK_URL not set!")
# In-memory store for processed event IDs (idempotency) # In-memory store for processed event IDs (idempotency)
processed_event_ids = set() processed_event_ids = set()
MAX_STORED_EVENT_IDS = 10000 MAX_STORED_EVENT_IDS = 10000
@@ -65,10 +53,11 @@ async def get_stats():
""" """
Get daily statistics for LEGO inventory. Get daily statistics for LEGO inventory.
Returns metrics about enrichment progress, missing data, and catalog value. Returns metrics about enrichment progress, missing data, catalog value, and cache stats.
""" """
try: try:
daily_stats = await stats.get_daily_stats() daily_stats = await stats.get_daily_stats()
cache_stats = await stats.get_brickset_cache_stats()
# Calculate completion percentage # Calculate completion percentage
if daily_stats["total_items"] > 0: if daily_stats["total_items"] > 0:
@@ -80,8 +69,11 @@ async def get_stats():
return { return {
"status": "success", "status": "success",
"version": "0.3.0",
"git_commit": stats.get_git_commit(),
"stats": daily_stats, "stats": daily_stats,
"completion_rate": round(completion_rate, 1), "completion_rate": round(completion_rate, 1),
"brickset_cache": cache_stats,
} }
except Exception as e: except Exception as e:
return JSONResponse( return JSONResponse(