name changes

config/gunicorn.conf.py

@@ -23,11 +23,11 @@ loglevel = "info"
 access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(D)s'
 
 # Process naming
-proc_name = "studspider-webhook"
+proc_name = "studspider"
 
 # Server mechanics
 daemon = False  # Run in foreground (systemd will handle daemonization)
-pidfile = "./logs/webhook.pid"
+pidfile = "./logs/studspider.pid"
 umask = 0
 user = None  # Run as current user (systemd will set user)
 group = None

src/asquare.py

@@ -55,10 +55,14 @@ async def update_itm(lego_set: dict):
     """
     Asynchronously updates an item in the Square catalog based on the provided LEGO set information.
     Args:
-        lego_set (dict): A dictionary containing LEGO set details. Expected keys include:
-            - "sets": List of set dictionaries, each with "name", "number", "barcode" (with "UPC"), and "image" (with "thumbnailURL").
-            - "pw_name": The name associated with the LEGO set in another system.
-            - "set_adjustredprice": The adjusted price for the set.
+        lego_set (dict): A dictionary containing LEGO set details from abricksetapi. Expected keys include:
+            - "set_num": LEGO set number (e.g., "75192")
+            - "set_name": Official LEGO set name
+            - "set_theme": LEGO theme (e.g., "Star Wars")
+            - "set_barcode": UPC barcode
+            - "set_image": Thumbnail URL
+            - "pw_name": Formatted name for Square inventory (e.g., "75192 - STAR WARS UCS Millennium Falcon")
+            - "set_adjustredprice": Retail price with 10% markup applied
     Returns:
         bool: True if the update operation was successful.
     Raises:
@@ -66,10 +70,10 @@ async def update_itm(lego_set: dict):
         Exception: For errors during Square API interaction.
     """
 
-    set_name = lego_set["sets"][0]["name"]
-    set_number = lego_set["sets"][0]["number"]
-    set_upc = lego_set["sets"][0]["barcode"]["UPC"]
-    set_image = lego_set["sets"][0]["image"]["thumbnailURL"]
+    set_number = lego_set["set_num"]
+    set_name = lego_set["set_name"]
+    set_upc = lego_set["set_barcode"]
+    set_image = lego_set["set_image"]
     pw_name = lego_set["pw_name"]
     set_adjustedprice = lego_set["set_adjustredprice"]
 

src/get_sq_env.py

@@ -6,20 +6,17 @@ dotenv.load_dotenv()
 
 LEGO_CAT_ID = os.getenv("SQUARE_LEGO_CAT_ID")
 
-# SSL Certificate paths (optional - for direct HTTPS, not needed with nginx)
-SSL_CERT_PATH = os.getenv("SSL_CERT_PATH")
-SSL_KEY_PATH = os.getenv("SSL_KEY_PATH")
-
-# Square Webhook Configuration
-# Get signature key from Square Developer Dashboard → Webhooks
-SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("SQUARE_WEBHOOK_SIGNATURE_KEY")
 SQUARE_WEBHOOK_URL = os.getenv("SQUARE_WEBHOOK_URL")
 
 if os.getenv("SQUARE_ENV") == "production":
     SQUARE_APP_ID = os.getenv("PROD_SQUARE_APP_ID")
     SQUARE_TOKEN = os.getenv("PROD_SQUARE_TOKEN")
+    SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("PROD_SQUARE_WEBHOOK_SIGNATURE_KEY")
+    SQUARE_WEBHOOK_SUB_ID = os.getenv("PROD_SQUARE_WEBHOOK_SUB_ID")
     environment = SquareEnvironment.PRODUCTION
 else:
     SQUARE_APP_ID = os.getenv("SANDBOX_SQUARE_APP_ID")
     SQUARE_TOKEN = os.getenv("SANDBOX_SQUARE_TOKEN")
+    SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY")
+    SQUARE_WEBHOOK_SUB_ID = os.getenv("SANDBOX_SQUARE_WEBHOOK_SUB_ID")
     environment = SquareEnvironment.SANDBOX

src/stats.py

@@ -0,0 +1,298 @@
+"""
+Statistics gathering functions for daily_stats table.
+Queries Square catalog to collect metrics about LEGO inventory quality.
+"""
+
+import json
+from typing import Dict, Any
+from square import AsyncSquare
+import get_sq_env
+
+# Initialize Square client
+client = AsyncSquare(
+    token=get_sq_env.SQUARE_TOKEN,
+    environment=get_sq_env.environment
+)
+
+LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
+
+
+async def get_daily_stats() -> Dict[str, Any]:
+    """
+    Gather all metrics for daily_stats table in a single pass.
+    
+    Returns:
+        dict: {
+            "total_items": int,
+            "enriched_items": int,
+            "missing_upcs": int,
+            "missing_images": int,
+            "total_catalog_value": float
+        }
+    """
+    # Get all items in LEGO category
+    result = await client.catalog.search(
+        object_types=["ITEM"],
+        query={
+            "exact_query": {
+                "attribute_name": "category_id",
+                "attribute_value": LEGO_CAT_ID,
+            }
+        },
+    )
+    
+    result_json = json.loads(result.model_dump_json())
+    objects = result_json.get("objects", [])
+    
+    # Initialize counters
+    total_items = 0
+    enriched_items = 0
+    missing_upcs = 0
+    missing_images = 0
+    total_catalog_value = 0.0
+    
+    for obj in objects:
+        total_items += 1
+        item_data = obj.get("item_data", {})
+        
+        # Check if item has all enrichment data
+        has_upc = False
+        has_image = False
+        has_proper_name = False
+        item_price = 0.0
+        
+        # Check variations for UPC and price
+        variations = item_data.get("variations", [])
+        if variations:
+            var_data = variations[0].get("item_variation_data", {})
+            
+            # Check UPC
+            if var_data.get("upc"):
+                has_upc = True
+            else:
+                missing_upcs += 1
+            
+            # Get price (stored in cents)
+            price_money = var_data.get("price_money", {})
+            if price_money and price_money.get("amount"):
+                item_price = price_money["amount"] / 100.0  # Convert cents to dollars
+                total_catalog_value += item_price
+        else:
+            missing_upcs += 1
+        
+        # Check for image
+        if item_data.get("image_ids"):
+            has_image = True
+        else:
+            missing_images += 1
+        
+        # Check if name follows format: "12345 - THEME Set Name"
+        name = item_data.get("name", "")
+        if " - " in name and name.split()[0].isdigit() and len(name.split()[0]) == 5:
+            has_proper_name = True
+        
+        # Item is "enriched" if it has UPC, image, and proper name format
+        if has_upc and has_image and has_proper_name:
+            enriched_items += 1
+    
+    return {
+        "total_items": total_items,
+        "enriched_items": enriched_items,
+        "missing_upcs": missing_upcs,
+        "missing_images": missing_images,
+        "total_catalog_value": round(total_catalog_value, 2),
+    }
+
+
+async def get_total_items() -> int:
+    """
+    Count total items in LEGO category.
+    
+    Returns:
+        int: Total number of items
+    """
+    result = await client.catalog.search(
+        object_types=["ITEM"],
+        query={
+            "exact_query": {
+                "attribute_name": "category_id",
+                "attribute_value": LEGO_CAT_ID,
+            }
+        },
+    )
+    
+    result_json = json.loads(result.model_dump_json())
+    return len(result_json.get("objects", []))
+
+
+async def get_enriched_count() -> int:
+    """
+    Count items with complete enrichment (UPC + image + proper name format).
+    
+    Returns:
+        int: Number of fully enriched items
+    """
+    result = await client.catalog.search(
+        object_types=["ITEM"],
+        query={
+            "exact_query": {
+                "attribute_name": "category_id",
+                "attribute_value": LEGO_CAT_ID,
+            }
+        },
+    )
+    
+    result_json = json.loads(result.model_dump_json())
+    objects = result_json.get("objects", [])
+    
+    enriched_count = 0
+    
+    for obj in objects:
+        item_data = obj.get("item_data", {})
+        
+        # Check UPC
+        has_upc = False
+        variations = item_data.get("variations", [])
+        if variations:
+            var_data = variations[0].get("item_variation_data", {})
+            has_upc = bool(var_data.get("upc"))
+        
+        # Check image
+        has_image = bool(item_data.get("image_ids"))
+        
+        # Check name format: "12345 - THEME Set Name"
+        name = item_data.get("name", "")
+        has_proper_name = False
+        if " - " in name:
+            first_part = name.split()[0]
+            if first_part.isdigit() and len(first_part) == 5:
+                has_proper_name = True
+        
+        if has_upc and has_image and has_proper_name:
+            enriched_count += 1
+    
+    return enriched_count
+
+
+async def get_missing_upcs() -> int:
+    """
+    Count items without UPC barcodes.
+    
+    Returns:
+        int: Number of items missing UPCs
+    """
+    result = await client.catalog.search(
+        object_types=["ITEM"],
+        query={
+            "exact_query": {
+                "attribute_name": "category_id",
+                "attribute_value": LEGO_CAT_ID,
+            }
+        },
+    )
+    
+    result_json = json.loads(result.model_dump_json())
+    objects = result_json.get("objects", [])
+    
+    missing_count = 0
+    
+    for obj in objects:
+        item_data = obj.get("item_data", {})
+        variations = item_data.get("variations", [])
+        
+        if not variations:
+            missing_count += 1
+            continue
+        
+        var_data = variations[0].get("item_variation_data", {})
+        if not var_data.get("upc"):
+            missing_count += 1
+    
+    return missing_count
+
+
+async def get_missing_images() -> int:
+    """
+    Count items without images.
+    
+    Returns:
+        int: Number of items missing images
+    """
+    result = await client.catalog.search(
+        object_types=["ITEM"],
+        query={
+            "exact_query": {
+                "attribute_name": "category_id",
+                "attribute_value": LEGO_CAT_ID,
+            }
+        },
+    )
+    
+    result_json = json.loads(result.model_dump_json())
+    objects = result_json.get("objects", [])
+    
+    missing_count = 0
+    
+    for obj in objects:
+        item_data = obj.get("item_data", {})
+        if not item_data.get("image_ids"):
+            missing_count += 1
+    
+    return missing_count
+
+
+async def get_total_catalog_value() -> float:
+    """
+    Calculate total dollar value of all items in LEGO category.
+    
+    Returns:
+        float: Total catalog value in dollars
+    """
+    result = await client.catalog.search(
+        object_types=["ITEM"],
+        query={
+            "exact_query": {
+                "attribute_name": "category_id",
+                "attribute_value": LEGO_CAT_ID,
+            }
+        },
+    )
+    
+    result_json = json.loads(result.model_dump_json())
+    objects = result_json.get("objects", [])
+    
+    total_value = 0.0
+    
+    for obj in objects:
+        item_data = obj.get("item_data", {})
+        variations = item_data.get("variations", [])
+        
+        if variations:
+            var_data = variations[0].get("item_variation_data", {})
+            price_money = var_data.get("price_money", {})
+            
+            if price_money and price_money.get("amount"):
+                # Price is stored in cents, convert to dollars
+                total_value += price_money["amount"] / 100.0
+    
+    return round(total_value, 2)
+
+
+# Example usage
+if __name__ == "__main__":
+    import asyncio
+    
+    async def main():
+        print("Gathering daily statistics for LEGO catalog...")
+        
+        stats = await get_daily_stats()
+        
+        print(f"\n=== Daily Stats ===")
+        print(f"Total Items:          {stats['total_items']}")
+        print(f"Enriched Items:       {stats['enriched_items']}")
+        print(f"Missing UPCs:         {stats['missing_upcs']}")
+        print(f"Missing Images:       {stats['missing_images']}")
+        print(f"Total Catalog Value:  ${stats['total_catalog_value']:,.2f}")
+        print(f"Completion Rate:      {(stats['enriched_items'] / stats['total_items'] * 100):.1f}%")
+    
+    asyncio.run(main())

studspider.py

@@ -4,19 +4,14 @@ import asyncio
 from typing import Dict, Any
 from fastapi import FastAPI, Request, Response, status
 from fastapi.responses import JSONResponse
-import dotenv
-
-# Load environment variables
-dotenv.load_dotenv()
+import src.get_sq_env as sq_env
 
 # Square webhook validation
 from square.utils.webhooks_helper import verify_signature
 
 # Configuration
-SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("SANDBOX_SQUARE_SIGNATURE_KEY") or os.getenv(
-    "PROD_SQUARE_SIGNATURE_KEY"
-)
-SQUARE_WEBHOOK_URL = os.getenv("SQUARE_WEBHOOK_URL")
+SQUARE_WEBHOOK_SIGNATURE_KEY = sq_env.SQUARE_WEBHOOK_SIGNATURE_KEY
+SQUARE_WEBHOOK_URL = sq_env.SQUARE_WEBHOOK_URL
 
 # Debug: Print configuration (redact key)
 if SQUARE_WEBHOOK_SIGNATURE_KEY: