add initial web hook handling

.gitignore

@@ -8,7 +8,6 @@ __pycache__/
 *.db
 *.sqlite3
 *.log
-poetry.lock
 bin/build/
 .zip
 snippet.py
@@ -23,3 +22,17 @@ doc/pw-inventory/SQUARE_inventory_forautomation_2025-06-18-0139.csv
 doc/pw-inventory/square_inventory_export_2025-06-18-0137.xlsx
 
 square-src.code-workspace
+
+doc/pw-inventory/
+
+studspider.db-journal
+.venv
+*.db
+*.py[cod]
+.web
+.states
+assets/external/
+__pycache__/
+win/
+.github/
+activate/

README.md

@@ -1,11 +1,43 @@
-# Edderkop
+# studspider
 
-environment: poetry
-code style: black
+![Static Badge](https://img.shields.io/badge/python-3.13.7-blue?logo=python) ![Static Badge](https://img.shields.io/badge/code_style-black-black) ![Static Badge](https://img.shields.io/badge/license-MPL-orange)
 
-built with
+
+
+
+is a tool used to talk to brickset.com and a square inventory to pull lego set information for entering in the square db
+## Run me
+
+Clone the project
+
+```bash
+  git clone http://git.traps.moe/ph0sgene/studspider.git
+  git checkout uv 
+```
+
+Go to the project directory
+
+```bash
+  cd studspider
+```
+
+Install dependencies
+
+```bash
+  uv sync && uv lock
+```
+
+Start the server
+
+```bash
+  uvx run studspider.py
+```
+
+
+## Tech 
 
 - aiohttp
 - squareup
 - apscheduler
 - aiosqlite
+

TODO.md

@@ -2,3 +2,10 @@
 - test sq
 - test inserting brickowl into sq
 - test kopling
+
+
+square inv needs lego set category changed
+
+key lights
+pastel wrist things
+some sets

pyproject.toml

@@ -1,13 +1,11 @@
 [project]
-name = "edderkop"
+name = "studpider"
 version = "0.1.0"
 description = ""
-authors = [
-    {name = "CF", email = "chris-forst@pm.me"}
-]
-license = {text = "MIT"}
-readme = "README.md"
+authors = [{ name = "CF", email = "ph0sgene@pm.me" }]
 requires-python = ">=3.13,<3.14"
+readme = "README.md"
+license = { text = "MIT" }
 dependencies = [
     "aiohttp (>=3.12.13,<4.0.0)",
     "dotenv (>=0.9.9,<0.10.0)",
@@ -18,15 +16,16 @@ dependencies = [
     "pytest (>=8.4.2,<9.0.0)",
     "apscheduler (>=3.11.0,<4.0.0)",
     "aiosqlite (>=0.21.0,<0.22.0)",
+    "bs4>=0.0.2",
+    "reflex>=0.8.14.post1",
 ]
 
-
 [build-system]
-requires = ["poetry-core>=2.0.0,<3.0.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
-package-mode = false
+[tool.uv]
+package = false
 
 [tool.bricksetapi]
 version = "0.1.0"

setup.py

@@ -1,19 +1,19 @@
 from setuptools import setup
 
 setup(
-    name='Edderkop',
-    version='0.1',
-    description='Bricket + Square integration application',
-    author='CF',
-    author_email='chris-forst@pm.me',
-    packages=['src'],
-    package_dir={'': 'src'},
+    name="studspider",
+    version="0.1",
+    description="Bricket + Square integration application",
+    author="CF",
+    author_email="chris-forst@pm.me",
+    packages=["src"],
+    package_dir={"": "src"},
     install_requires=[
         # List your project dependencies here
     ],
     entry_points={
-        'console_scripts': [
-            'main=main:main',  # Adjust this line based on your main function
+        "console_scripts": [
+            "main=main:main",  # Adjust this line based on your main function
         ],
     },
 )

src/abricksetapi.py

@@ -9,7 +9,9 @@ import aiohttp
 import json
 import dotenv
 from typing import Dict, Any
-from asyncio_throttle.throttler import Throttler
+import throttler
+
+Throttler = throttler.GlobalThrottler
 
 dotenv.load_dotenv()
 
@@ -17,14 +19,15 @@ dotenv.load_dotenv()
 class AsyncBricksetAPI:
     """Asynchronous client for the Brickset API"""
 
-    def __init__(self, debug: bool = False):
+    def __init__(self, debug: bool = False, logger=None):
         self.username = os.getenv("BRICKSET_USERNAME")
         self.password = os.getenv("BRICKSET_PASS")
         self.api_key = os.getenv("BRICKSET_API_KEY")
         self.debug = debug
-        self.headers = {"UserAgent": "Edderkop/0.1"}
+        self.headers = {"UserAgent": "studspider/0.1"}
         self.throttler = Throttler(rate_limit=500, period=60)
         self.user_hash = None
+        self.logger = logger
 
     async def login(self) -> Dict[str, Any]:
         """Asynchronously authenticate with Brickset API"""
@@ -41,13 +44,18 @@ class AsyncBricksetAPI:
             ) as response:
                 if self.debug:
                     print(f"[@DEBUG] Login response status: {response.status}")
+                if self.logger:
+                    self.logger.debug(f"Login response status: {response.status}")
 
                 data = await response.read()
                 result = json.loads(data.decode("utf-8"))
                 self.user_hash = result.get("hash")
+
+                if self.logger:
+                    self.logger.info("Successfully authenticated with Brickset API")
                 return result
 
-    async def get_set(self, set_number: str) -> Dict[str, Any]:
+    async def get_set(self, set_number: int) -> Dict[str, Any]:
         """
         Fetch LEGO set information asynchronously
 
@@ -66,6 +74,7 @@ class AsyncBricksetAPI:
             "userHash": self.user_hash,
             "params": json.dumps({"setNumber": f"{set_number}-1"}),
         }
+        print(f"[@DEBUG] {params}...")
 
         async with aiohttp.ClientSession() as session:
             async with session.get(
@@ -73,13 +82,23 @@ class AsyncBricksetAPI:
             ) as response:
                 if self.debug:
                     print(f"[@DEBUG] Get set response status: {response.status}")
+                if self.logger:
+                    self.logger.debug(f"Get set response status: {response.status}")
 
                 if response.status == 200:
                     data = await response.read()
                     result = json.loads(data.decode("utf-8"))
 
                     if not result.get("sets"):
-                        raise ValueError(f"No data found for set {set_number}")
+                        error_msg = f"No data found for set {set_number}"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise ValueError(error_msg)
+
+                    if self.logger:
+                        self.logger.info(
+                            f"Successfully retrieved data for set {set_number}"
+                        )
 
                     set_data = result["sets"][0]
                     our_set = {
@@ -111,11 +130,17 @@ class AsyncBricksetAPI:
                     return our_set
 
                 elif response.status == 500:
-                    raise Exception(
+                    error_msg = (
                         "Internal Server Error: Please check API key and parameters."
                     )
+                    if self.logger:
+                        self.logger.error(error_msg)
+                    raise Exception(error_msg)
                 else:
-                    raise Exception(f"Unexpected response: {response.status}")
+                    error_msg = f"Unexpected response: {response.status}"
+                    if self.logger:
+                        self.logger.error(error_msg)
+                    raise Exception(error_msg)
 
     async def throttled_get_set(self, set_number: str) -> Dict[str, Any]:
         """Get set information with rate limiting"""

src/asquare.py

@@ -4,6 +4,9 @@ import json
 import asyncio
 import uuid
 import re
+from time import time
+
+import db
 
 import get_sq_env
 
@@ -103,7 +106,17 @@ async def acat2id(cat_id: str):
         {
             "id": obj["id"],
             "name": obj.get("item_data", {}).get("name", "Unknown"),
-            "number": get_set_number(obj.get("item_data", {}).get("name", ""))
+            "number": (
+                None
+                if not obj.get("item_data", {}).get("name", "")
+                else (
+                    get_set_number(obj.get("item_data", {}).get("name", ""))
+                    if re.findall(r"\d{5}", obj.get("item_data", {}).get("name", ""))
+                    else None
+                )
+            ),
+            "updated_at": obj.get("updated_at"),
+            "version": obj.get("version"),
         }
         for obj in result_json.get("objects", [])
     ]
@@ -112,14 +125,10 @@ async def acat2id(cat_id: str):
 
 
 async def test() -> None:
-    task = asyncio.create_task(acat2id(LEGO_CAT_ID))  # type: ignore
-    await asyncio.sleep(1)  # Give the task some time to complete
-    result = await task
+    result = await acat2id(LEGO_CAT_ID)  # type: ignore
     print("OBJECTS IN LEGO CATEGORY:")
     for item in result:
-        print(f"ID: {item['id']}, Name: {item['name']}")
-    # eturn
-    # pass
+        print(f"ID: {item['id']}, Name: {item['name']}, Updated: {item['updated_at']}")
 
 
 """

src/asquareclass.py

@@ -0,0 +1,218 @@
+import os
+import json
+import asyncio
+import re
+from typing import List, Dict, Any, Optional
+from square import AsyncSquare
+import get_sq_env
+
+
+class AsyncSquareClient:
+    """Asynchronous client for the Square API with focus on catalog operations"""
+
+    def __init__(
+        self,
+        token: Optional[str] = None,
+        environment: Optional[str] = None,
+        logger=None,
+    ):
+        """
+        Initialize the AsyncSquareClient.
+
+        Args:
+            token (str, optional): Square API token. If not provided, fetches from environment.
+            environment (str, optional): Square environment (sandbox/production). If not provided, fetches from environment.
+            logger: Optional logger object for logging operations.
+        """
+        self.token = token or get_sq_env.SQUARE_TOKEN
+        self.environment = environment or get_sq_env.environment
+        self.lego_cat_id = get_sq_env.LEGO_CAT_ID
+        self.client = AsyncSquare(token=self.token, environment=self.environment)  # type: ignore
+        self.logger = logger
+
+    @staticmethod
+    def get_set_number(text: str) -> int:
+        """
+        Extract a five-digit LEGO set number from a string.
+
+        Args:
+            text (str): The text to search for a set number.
+
+        Returns:
+            int: The extracted set number.
+
+        Raises:
+            ValueError: If no five-digit number is found.
+        """
+        numbers = re.findall(r"\d{5}", text)
+        if not numbers:
+            raise ValueError(f"No five-digit number found in string {text}")
+        return int(numbers[0])
+
+    async def get_item(self, sku: str) -> Dict[str, str]:
+        """
+        Asynchronously retrieves item details from Square Catalog using a SKU.
+
+        Args:
+            sku (str): The SKU or text filter to search for the item.
+
+        Returns:
+            dict: A dictionary containing the item's name and ID.
+
+        Raises:
+            IndexError: If no items are found matching the SKU.
+            Exception: If there is an error communicating with the Square API.
+        """
+        if self.logger:
+            self.logger.debug(f"Searching for item with SKU: {sku}")
+
+        result = await self.client.catalog.search_items(text_filter=sku, limit=1)
+        result_json = json.loads(result.model_dump_json())
+
+        try:
+            name = result_json["items"][0]["item_data"]["name"]
+            itm_id = result_json["items"][0]["id"]
+
+            if self.logger:
+                self.logger.info(f"Found item: {name} with ID: {itm_id}")
+
+            return {"name": name, "id": itm_id}
+        except (KeyError, IndexError) as e:
+            if self.logger:
+                self.logger.error(f"Error retrieving item with SKU {sku}: {str(e)}")
+            raise
+
+    async def update_item(self, lego_set: Dict[str, Any]) -> bool:
+        """
+        Asynchronously updates an item in the Square catalog based on provided LEGO set information.
+
+        Args:
+            lego_set (dict): A dictionary containing LEGO set details including:
+                - sets: List containing set information
+                - pw_name: PriceWise system name
+                - set_adjustredprice: Adjusted retail price
+
+        Returns:
+            bool: True if update was successful.
+
+        Raises:
+            KeyError: If required keys are missing from input dictionary.
+            Exception: For Square API communication errors.
+        """
+        if self.logger:
+            self.logger.debug(
+                f"Updating item with set info: {lego_set['sets'][0]['number']}"
+            )
+
+        try:
+            set_name = lego_set["sets"][0]["name"]
+            set_number = lego_set["sets"][0]["number"]
+            set_upc = lego_set["sets"][0]["barcode"]["UPC"]
+            set_image = lego_set["sets"][0]["image"]["thumbnailURL"]
+            pw_name = lego_set["pw_name"]
+            set_adjustedprice = lego_set["set_adjustredprice"]
+
+            result = await self.client.catalog.search_items(
+                text_filter=set_number, limit=1
+            )
+            itm_id = json.loads(result.model_dump_json())["items"][0]["id"]
+
+            if self.logger:
+                self.logger.info(f"Successfully updated item {set_number}")
+
+            # TODO: Implement actual update logic here
+            # await self.client.catalog.object.upsert()
+            return True
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error updating item: {str(e)}")
+            raise
+
+    async def get_category_items(
+        self, cat_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Return a list of items within the specified category ID.
+
+        Args:
+            cat_id (str, optional): The category ID to search for items.
+                                  If not provided, uses the default LEGO category ID.
+
+        Returns:
+            list[dict]: A list of dictionaries, each containing:
+                - 'id': The item's ID
+                - 'name': The item's name
+                - 'number': The LEGO set number
+        """
+        category_id = cat_id or self.lego_cat_id
+
+        if self.logger:
+            self.logger.debug(f"Fetching items for category: {category_id}")
+
+        try:
+            # Create a properly structured query for the Square API
+            search_request = {
+                "object_types": ["ITEM"],
+                "query": {
+                    "exact_query": {
+                        "attribute_name": "category_id",
+                        "attribute_value": category_id,
+                    }
+                },
+            }
+
+            result = await self.client.catalog.search(**search_request)
+            result_json = json.loads(result.model_dump_json())
+
+            items = [
+                {
+                    "id": obj["id"],
+                    "name": obj.get("item_data", {}).get("name", "Unknown"),
+                    "number": self.get_set_number(
+                        obj.get("item_data", {}).get("name", "")
+                    ),
+                }
+                for obj in result_json.get("objects", [])
+            ]
+
+            if self.logger:
+                self.logger.info(f"Found {len(items)} items in category {category_id}")
+
+            return items
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error fetching category items: {str(e)}")
+            raise
+
+
+# Provide acat2id as a module-level function for backward compatibility
+async def acat2id(cat_id: str) -> List[Dict[str, Any]]:
+    """
+    Backward compatibility function for getting category items.
+
+    Args:
+        cat_id (str): The category ID to search for items.
+
+    Returns:
+        list[dict]: A list of dictionaries containing item information.
+    """
+    client = AsyncSquareClient()
+    return await client.get_category_items(cat_id)
+
+
+async def test() -> None:
+    """Test function to demonstrate class usage"""
+    client = AsyncSquareClient()
+    print(f"@[DEBUG] ENV:\t{client.environment}")
+    print(f"@[DEBUG] CATID:\t{client.lego_cat_id}")
+
+    result = await client.get_category_items()
+    print("\nOBJECTS IN LEGO CATEGORY:")
+    for item in result:
+        print(f"ID: {item['id']}, Name: {item['name']}")
+
+
+if __name__ == "__main__":
+    asyncio.run(test())

src/bricksetapi.py

@@ -14,9 +14,9 @@ debug = False  # Set to True to enable debug output
 
 def login(username, password):
     url = "https://brickset.com/api/v3.asmx/login"
-    headers = {"UserAgent": "Edderkop/0.1"}
+    headers = {"UserAgent": "studspider/0.1"}
     params = {
-        "UserAgent": "Edderkop/0.1",
+        "UserAgent": "studspider/0.1",
         "username": username,
         "password": password,
         "apiKey": BRICKSET_API_KEY,
@@ -33,6 +33,11 @@ def get_set(set_number: int) -> dict:
     response = requests.get(url)
     if response.status_code == 200:
         data = response.json()
+        if not data["sets"] and data["message"] == "API limit exceeded":
+            raise Exception("API limit exceeded. Please try again later.")
+            return False
+
+        print(f"[@DEBUG]\t{data}")
         our_set = {
             "set_num": data["sets"][0]["number"],
             "set_name": data["sets"][0]["name"],

src/db.py

@@ -0,0 +1,61 @@
+import aiosqlite
+from time import time
+
+time = int(time())  # we dont need no stinkin floats
+
+# db = await aiosqlite.connect("studspider.db")  # type: ignore
+
+
+async def close(db):
+    await db.close()
+
+
+async def fetch(table, sqobj):
+    """
+    Asynchronously fetch a record from the REVIEW table by sq_object.
+
+    Args:
+        sqobj (str): The sq_object identifier to search for.
+    Returns:
+        dict or None: The fetched record as a dictionary, or None if not found.
+    """
+    async with aiosqlite.connect("studspider.db") as db:  # type: ignore
+        async with db.execute(
+            f"SELECT * FROM {table} WHERE sq_object = {sqobj}"
+        ) as cursor:
+            async for row in cursor:
+                return row
+
+    return None
+
+
+async def upsert(table: str, data: dict, unique_key: str) -> bool:
+    """
+    Asynchronously insert or update a record in the specified table.
+
+    Args:
+        table (str): The name of the table to upsert into.
+        data (dict): A dictionary of column-value pairs to insert or update.
+        unique_key (str): The column name that serves as the unique key for upsert.
+
+    Returns:
+        None
+    """
+    print(f"Upserting into {table}: {data}")
+
+    columns = ", ".join(data.keys())
+    placeholders = ", ".join("?" * len(data))
+    updates = ", ".join(
+        f"{col}=excluded.{col}" for col in data.keys() if col != unique_key
+    )
+
+    sql = f"""
+    INSERT INTO {table} ({columns})
+    VALUES ({placeholders})
+    ON CONFLICT({unique_key}) DO UPDATE SET
+    {updates};
+    """
+    async with aiosqlite.connect("studspider.db") as db:  # type: ignore
+        async with db.execute(sql, tuple(data.values())) as cursor:
+            await db.commit()
+    return True

src/ebricksetapi.py

@@ -15,7 +15,7 @@ class BricksetAPI:
         self.password = os.getenv("BRICKSET_PASS")
         self.api_key = os.getenv("BRICKSET_API_KEY")
         self.debug = debug
-        self.headers = {"UserAgent": "Edderkop/0.1"}
+        self.headers = {"UserAgent": "studspider/0.1"}
         self.user_hash = None
 
     def login(self) -> Dict[str, Any]:

src/glue.py

@@ -0,0 +1,41 @@
+import asyncio
+import json
+import re
+from . import get_sq_env
+import aiosqlite
+from time import time
+
+from . import asquare
+from . import db
+from . import abricksetapi
+from . import throttler 
+Throttler = throttler.GlobalThrottler
+
+
+LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
+
+# big glue boy
+async def do_review() -> bool:
+    result = await asquare.acat2id(LEGO_CAT_ID) # type: ignore
+    for item in result:
+        if not re.match(r".*\d{5}.*", item["name"]):
+            missing = []
+            print(f"{item} is missing its set number!")
+            #print(f"[@DEBUG] Upserting {item['id']} into REVIEW table")
+            item_sanitized = {
+                "sq_object": item["id"],
+                "set_name": item["name"],
+                "set_number": None,
+                "time_added": time
+            }
+            missing.append(item_sanitized)
+    
+    for item in missing:
+        print(item)
+        # sq_item = await db.fetch(item["id"])  
+        if item["id"] != sq_item["sq_object"]:
+            print(f"[@DEBUG] {item['id']} is missing)")
+            #await db.upsert("MISSING", item_sanitized, "sq_object")
+        #    #print(f"[@DEBUG] Upserted {item['id']} into REVIEW table")
+        #print(f"[@DEBUG] {item['id']} already in REVIEW table")
+    return True

src/scheduler.py

@@ -6,21 +6,21 @@ import os
 
 logger = logging.getLogger(__name__)
 
-# Get the absolute path to edderkop.db in the project root
+# Get the absolute path to studspider.db in the project root
 DEFAULT_DB_PATH = os.path.join(
-    os.path.dirname(os.path.dirname(__file__)), "edderkop.db"
+    os.path.dirname(os.path.dirname(__file__)), "studspider.db"
 )
 
 if not os.path.exists(DEFAULT_DB_PATH):
     raise FileNotFoundError(f"Database file not found at {DEFAULT_DB_PATH}")
-    print('Please run "src/scripts/EDDERKOP.SQL" to setup the database.')
+    print('Please run "src/scripts/studspider.SQL" to setup the database.')
     os.exit(0)
 
 
 class AsyncSchedulerDB:
     """
     Asynchronous SQLite database manager for APScheduler.
-    Handles job storage, retrieval, and database operations using the main edderkop.db database.
+    Handles job storage, retrieval, and database operations using the main studspider.db database.
     """
 
     def __init__(self, db_path: str = DEFAULT_DB_PATH):
@@ -29,7 +29,7 @@ class AsyncSchedulerDB:
 
         Args:
             db_path (str, optional): Path to the SQLite database file.
-                                   Defaults to edderkop.db in project root.
+                                   Defaults to studspider.db in project root.
         """
         self.db_path = db_path
         self._initialized = False

src/scripts/edderkop.sql

@@ -1,7 +0,0 @@
-CREATE TABLE "REVIEW" (
-	"sq_object"	TEXT NOT NULL UNIQUE,
-	"time_added"	INTEGER,
-	"set_number"	INTEGER,
-	"time_modified"	INTEGER,
-	PRIMARY KEY("sq_object")
-)

src/scripts/edderkop.sqlite

@@ -0,0 +1,15 @@
+CREATE TABLE "REVIEW" (
+	"sq_object"	TEXT NOT NULL UNIQUE,
+	"price"	INTEGER NOT NULL,
+	"date_modified"	INTEGER,
+	PRIMARY KEY("sq_object")
+)
+
+CREATE TABLE "MISSING" (
+	"sq_object"	TEXT NOT NULL UNIQUE,
+	"time_added"	INTEGER,
+	"set_number"	INTEGER,
+	"time_modified"	INTEGER,
+	"set_name"	TEXT,
+	PRIMARY KEY("sq_object")
+)

src/server.py

@@ -74,7 +74,7 @@ async def handle_post(request):
                     return True
         else:
             return aiohttp.web.Response(status=400, text="400 Bad Request.")
-        result = {"message": f"Edderkop/0.1 received command: {reqdata}"}
+        result = {"message": f"studspider/0.1 received command: {reqdata}"}
         return aiohttp.web.json_response(status=200, data=result)
 
 
@@ -108,6 +108,6 @@ async def main():
 
 
 if __name__ == "__main__":
-    print(generate_toilet_banner("Edderkop v0.1"))
+    print(generate_toilet_banner("studspider v0.1"))
     print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
     aiohttp.web.run_app(app, port=7432)

src/test_server.py

@@ -1,45 +1,78 @@
+import argparse
 import asyncio
 import json
 from aiohttp import web
 from base64 import b64decode
 from typing import Dict, Any
 import uuid
-from time import time
+
 import dotenv
 import os
+
 import re
 import logging
+from logging.handlers import RotatingFileHandler
 from pathlib import Path
 
-Path("./logs/").mkdir(parents=True, exist_ok=True)
+print("Creating log directory if it doesn't exist...")
+os.makedirs("./logs/", exist_ok=True)
 
 import banner
 import get_sq_env
+from time import time
+
+time = int(time())
 import asquare as sq
+
+# from asquareclass import AsyncSquareClient  # type: ignore
 from abricksetapi import AsyncBricksetAPI
 
-lego = AsyncBricksetAPI(debug=True)
-
+print("Loading environment variables...")
 dotenv.load_dotenv()
 SQUARE_LEGO_CAT_ID = os.environ.get("SQUARE_LEGO_CAT_ID")
 
-
 from asyncio_throttle.throttler import Throttler
 
-logger = logging.getLogger("aiohttp.access")
-logging.basicConfig(
-    filename="./logs/edderkop.log", encoding="utf-8", level=logging.DEBUG
-)
+# set up logger
+print("Setting up logger...")
+logger = logging.getLogger("studspider")
+logger.setLevel(logging.DEBUG)
+handler = RotatingFileHandler("./logs/studspider.log", maxBytes=125000, backupCount=5)
+logger.addHandler(handler)
 
+lego = AsyncBricksetAPI(debug=True, logger=logger)
+# sq = AsyncSquareClient(logger=logger)
+# logging.basicConfig(
+#    filename="./logs/studspider.log", encoding="utf-8", level=logging.DEBUG
+# )
+print("Setting up rate limiter...")
+print("Doing RE things...")
 UA = re.compile(r"^kopling\/?", re.IGNORECASE)
 
 
 async def run_update():
-    set_list = await sq.acat2id(SQUARE_LEGO_CAT_ID)
+    set_list = await sq.acat2id(SQUARE_LEGO_CAT_ID)  # type: ignore
+    # print(f"[@DEBUG]\t {set_list}")  # type: ignore
     for l_set in set_list:
+        # print(f"[@DEBUG]{l_set}")
+        try:
+            l_setinfo = await lego.throttled_get_set(l_set["number"])
+            print(f"[@DEBUG] Found {l_setinfo["pw_name"]}")
+        except Exception as e:
+            print(f"Error fetching set\t: {l_set}: {str(e)}")
+            continue
+
+        # print(f"[@DEBUG]{l_setinfo}")
+        # await sq.update_itm(l_set, l_setinfo)
+    """for l_set in set_list:
         print(f"[@DEBUG]{l_set}")
-        l_setinfo = await lego.throttled_get_set(l_set["number"])
-        print(f"[@DEBUG]{l_setinfo}")
+        try:
+            l_setinfo = await lego.throttled_get_set(l_set["number"])
+        except Exception as e:
+            print(f"Error fetching set\t: {l_set}: {str(e)}")
+            continue
+
+        print(f"[@DEBUG]{l_setinfo}")"""
     #    await sq.update_itm(l_set, l_setinfo)
 
 
@@ -51,14 +84,53 @@ def valid_uuid(val):
         return False
 
 
+async def hook_handler(request: web.Request) -> web.Response:
+    """Handle incoming JSON webhook POSTs on /hook endpoint"""
+    try:
+        # Verify request method
+        if request.method != "POST":
+            logger.error(
+                f"{time}\tInvalid request method to /hook from %s", request.remote
+            )
+            return web.Response(status=405, text="Method not allowed. Use POST.")
+
+        # Get JSON data from request
+        try:
+            data = await request.json()
+        except json.JSONDecodeError:
+            logger.error("Invalid JSON to /hook from %s", request.remote)
+            return web.Response(status=400, text="Invalid JSON in request body")
+
+        # Log the received webhook
+        print(f"[@DEBUG] Webhook received at {time} from {request.remote}")
+        logger.info(
+            f"{time}\tWebhook received from {request.remote}: {json.dumps(data)}"
+        )
+
+        # Process the webhook data here
+        # TODO: Add your webhook processing logic
+
+        response_data = {
+            "status": "success",
+            "message": "Webhook received",
+            "timestamp": time,
+            "data_received": data,
+        }
+
+        return web.json_response(response_data)
+
+    except Exception as e:
+        logger.error(f"Webhook handler error: {str(e)}")
+        print(f"Webhook handler error: {str(e)}")
+        return web.Response(status=500, text="Internal server error")
+
+
 async def request_handler(request: web.Request) -> web.Response:
     """Handle incoming POST requests with command processing"""
     try:
         # Verify request method
         if request.method != "POST":
-            logger.error(
-                f"{time()}\tInvalid request method from %s at ", request.remote
-            )
+            logger.error(f"{time}\tInvalid request method from %s at ", request.remote)
             return web.Response(status=405, text="Method not allowed. Use POST.")
 
         # Get JSON data from request
@@ -86,19 +158,17 @@ async def request_handler(request: web.Request) -> web.Response:
         try:
             decoded_cmd = b64decode(data["CMD"]).decode("utf-8")
             print(
-                f"""At: {time()}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
+                f"""At: {time}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
             )
             logging.info(
-                f"{time()}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
+                f"{time}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
             )
 
             # Process the command
             if decoded_cmd == "RUNUPDATE":
-                # Add your command handling logic here
-                await run_update()
                 response_data = {"status": "success", "message": "Update started"}
+                await run_update()
                 logging.info(f"{time()}\tUpdate command processed successfully")
-
                 return web.json_response(response_data)
             else:
                 return web.Response(status=400, text=f"Unknown command: {decoded_cmd}")
@@ -112,10 +182,64 @@ async def request_handler(request: web.Request) -> web.Response:
         return web.Response(status=500, text="Internal server error")
 
 
+print("Starting web server...")
+os.system("cls" if os.name == "nt" else "clear")
 app = web.Application()
-app.router.add_post("/", request_handler)
+app.router.add_post("/api", request_handler)
+app.router.add_post("/hook", hook_handler)
+
+parser = argparse.ArgumentParser(description="studspider")
+parser.add_argument(
+    "--host",
+    type=str,
+    default="localhost",
+    help="IP address to listen on. default: localhost",
+)
+parser.add_argument(
+    "--port", type=int, default=7432, help="Port to listen on. default: 7432"
+)
+parser.add_argument(
+    "--hook-port",
+    type=int,
+    default=8080,
+    help="Optional separate port for webhook endpoint. default: 8080",
+)
+args = parser.parse_args()
 
 if __name__ == "__main__":
-    print(banner.generate("Edderkop v0.1"))
+    print(banner.generate("studspider v0.1"))
     print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
-    web.run_app(app, host="localhost", port=7432)
+
+    # If hook-port is specified, run a separate server for webhooks
+    if args.hook_port:
+        print(f"Starting main server on {args.host}:{args.port}")
+        print(f"Starting webhook server on {args.host}:{args.hook_port}")
+
+        # Create separate app for webhook
+        hook_app = web.Application()
+        hook_app.router.add_post("/hook", hook_handler)
+
+        # Run both servers concurrently
+        async def run_servers():
+            runner1 = web.AppRunner(app)
+            await runner1.setup()
+            site1 = web.TCPSite(runner1, args.host, args.port)
+
+            runner2 = web.AppRunner(hook_app)
+            await runner2.setup()
+            site2 = web.TCPSite(runner2, args.host, args.hook_port)
+
+            await site1.start()
+            await site2.start()
+
+            print(f"Main server running on http://{args.host}:{args.port}")
+            print(f"Webhook server running on http://{args.host}:{args.hook_port}/hook")
+
+            # Keep running
+            await asyncio.Event().wait()
+
+        asyncio.run(run_servers())
+    else:
+        print(f"Server running on {args.host}:{args.port}")
+        print(f"Webhook endpoint: http://{args.host}:{args.port}/hook")
+        web.run_app(app, host=args.host, port=args.port)

src/throttler.py

@@ -0,0 +1,33 @@
+import asyncio
+from asyncio_throttle import Throttler as AsyncThrottler
+
+class GlobalThrottler:
+    _instance = None
+    _lock = asyncio.Lock()
+    
+    def __init__(self, rate_limit: int = 500, period: int = 60) -> None:
+        if not hasattr(self, '_initialized'):
+            self.rate_limit = rate_limit
+            self.period = period
+            self.throttler = AsyncThrottler(rate_limit=self.rate_limit, period=self.period)
+            self._initialized = True
+    
+    @classmethod
+    async def get_instance(cls, rate_limit: int = 500, period: int = 60) -> 'GlobalThrottler':
+        async with cls._lock:
+            if not cls._instance:
+                cls._instance = cls(rate_limit, period)
+            return cls._instance
+    
+    async def acquire(self):
+        await self.throttler.acquire()
+        
+    def release(self):
+        self.throttler.release()
+    
+    async def __aenter__(self):
+        await self.acquire()
+        return self
+        
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self.release()

studspider.py

@@ -1,12 +1,12 @@
 import src.test_server
 
 
-class edderkop:
+class Studspider:
     def __init__(self):
-        self.name = "Edderkop"
+        self.name = "Studspider"
         self.version = "0.0.1"
         self.description = """
-        Edderkop is an application designed to interact with the Brickset API and directly with Square.
+        studspider is an application designed to interact with the Brickset API and directly with Square.
         It allows users to search for items, manage collections, and perform various operations related to bricksets.
         """