Compare commits
5
Commits
docker
..
b4b0ccdac6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b4b0ccdac6 | ||
|
|
32664c3a10 | ||
|
|
41ce94c85c | ||
|
|
397eddf299 | ||
|
|
8affb90308 |
No files matched your search
+2
-13
@@ -8,6 +8,8 @@ __pycache__/
|
||||
*.db
|
||||
*.sqlite3
|
||||
*.log
|
||||
poetry.lock
|
||||
uv.lock
|
||||
bin/build/
|
||||
.zip
|
||||
snippet.py
|
||||
@@ -22,16 +24,3 @@ doc/pw-inventory/SQUARE_inventory_forautomation_2025-06-18-0139.csv
|
||||
doc/pw-inventory/square_inventory_export_2025-06-18-0137.xlsx
|
||||
|
||||
square-src.code-workspace
|
||||
|
||||
doc/pw-inventory/
|
||||
|
||||
studspider.db-journal
|
||||
.venv
|
||||
*.db
|
||||
*.py[cod]
|
||||
.web
|
||||
.states
|
||||
assets/external/
|
||||
__pycache__/
|
||||
win/
|
||||
.github/
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
FROM python:3.13-alpine
|
||||
|
||||
# Install uv
|
||||
RUN pip install uv
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy project files
|
||||
COPY pyproject.toml ./
|
||||
COPY README.md ./
|
||||
|
||||
# Install dependencies using uv without venv
|
||||
RUN uv pip compile pyproject.toml -o requirements.txt && \
|
||||
uv pip install --system -r requirements.txt
|
||||
|
||||
# Copy the rest of the application
|
||||
COPY . .
|
||||
|
||||
# Default command
|
||||
CMD ["python", "tests/test.py"]
|
||||
@@ -1,43 +1,12 @@
|
||||
# studspider
|
||||
# Edderkop
|
||||
|
||||
  
|
||||
environment: poetry
|
||||
|
||||
code style: black
|
||||
|
||||
|
||||
|
||||
is a tool used to talk to brickset.com and a square inventory to pull lego set information for entering in the square db
|
||||
## Run me
|
||||
|
||||
Clone the project
|
||||
|
||||
```bash
|
||||
git clone http://git.traps.moe/ph0sgene/studspider.git
|
||||
git checkout uv
|
||||
```
|
||||
|
||||
Go to the project directory
|
||||
|
||||
```bash
|
||||
cd studspider
|
||||
```
|
||||
|
||||
Install dependencies
|
||||
|
||||
```bash
|
||||
uv sync && uv lock
|
||||
```
|
||||
|
||||
Start the server
|
||||
|
||||
```bash
|
||||
uvx run studspider.py
|
||||
```
|
||||
|
||||
|
||||
## Tech
|
||||
built with
|
||||
|
||||
- aiohttp
|
||||
- squareup
|
||||
- apscheduler
|
||||
- aiosqlite
|
||||
|
||||
@@ -2,10 +2,3 @@
|
||||
- test sq
|
||||
- test inserting brickowl into sq
|
||||
- test kopling
|
||||
|
||||
|
||||
square inv needs lego set category changed
|
||||
|
||||
key lights
|
||||
pastel wrist things
|
||||
some sets
|
||||
@@ -1,12 +1,12 @@
|
||||
import src.test_server
|
||||
|
||||
|
||||
class Studspider:
|
||||
class edderkop:
|
||||
def __init__(self):
|
||||
self.name = "Studspider"
|
||||
self.name = "Edderkop"
|
||||
self.version = "0.0.1"
|
||||
self.description = """
|
||||
studspider is an application designed to interact with the Brickset API and directly with Square.
|
||||
Edderkop is an application designed to interact with the Brickset API and directly with Square.
|
||||
It allows users to search for items, manage collections, and perform various operations related to bricksets.
|
||||
"""
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 68 KiB |
+1
-3
@@ -1,5 +1,5 @@
|
||||
[project]
|
||||
name = "studpider"
|
||||
name = "edderkop"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = [{ name = "CF", email = "ph0sgene@pm.me" }]
|
||||
@@ -16,8 +16,6 @@ dependencies = [
|
||||
"pytest (>=8.4.2,<9.0.0)",
|
||||
"apscheduler (>=3.11.0,<4.0.0)",
|
||||
"aiosqlite (>=0.21.0,<0.22.0)",
|
||||
"bs4>=0.0.2",
|
||||
"reflex>=0.8.14.post1",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
from setuptools import setup
|
||||
|
||||
setup(
|
||||
name="studspider",
|
||||
version="0.1",
|
||||
description="Bricket + Square integration application",
|
||||
author="CF",
|
||||
author_email="chris-forst@pm.me",
|
||||
packages=["src"],
|
||||
package_dir={"": "src"},
|
||||
name='Edderkop',
|
||||
version='0.1',
|
||||
description='Bricket + Square integration application',
|
||||
author='CF',
|
||||
author_email='chris-forst@pm.me',
|
||||
packages=['src'],
|
||||
package_dir={'': 'src'},
|
||||
install_requires=[
|
||||
# List your project dependencies here
|
||||
],
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"main=main:main", # Adjust this line based on your main function
|
||||
'console_scripts': [
|
||||
'main=main:main', # Adjust this line based on your main function
|
||||
],
|
||||
},
|
||||
)
|
||||
Whitespace-only changes.
+3
-6
@@ -9,9 +9,7 @@ import aiohttp
|
||||
import json
|
||||
import dotenv
|
||||
from typing import Dict, Any
|
||||
import throttler
|
||||
|
||||
Throttler = throttler.GlobalThrottler
|
||||
from asyncio_throttle.throttler import Throttler
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
@@ -24,7 +22,7 @@ class AsyncBricksetAPI:
|
||||
self.password = os.getenv("BRICKSET_PASS")
|
||||
self.api_key = os.getenv("BRICKSET_API_KEY")
|
||||
self.debug = debug
|
||||
self.headers = {"UserAgent": "studspider/0.1"}
|
||||
self.headers = {"UserAgent": "Edderkop/0.1"}
|
||||
self.throttler = Throttler(rate_limit=500, period=60)
|
||||
self.user_hash = None
|
||||
self.logger = logger
|
||||
@@ -55,7 +53,7 @@ class AsyncBricksetAPI:
|
||||
self.logger.info("Successfully authenticated with Brickset API")
|
||||
return result
|
||||
|
||||
async def get_set(self, set_number: int) -> Dict[str, Any]:
|
||||
async def get_set(self, set_number: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch LEGO set information asynchronously
|
||||
|
||||
@@ -74,7 +72,6 @@ class AsyncBricksetAPI:
|
||||
"userHash": self.user_hash,
|
||||
"params": json.dumps({"setNumber": f"{set_number}-1"}),
|
||||
}
|
||||
print(f"[@DEBUG] {params}...")
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(
|
||||
|
||||
+7
-16
@@ -4,9 +4,6 @@ import json
|
||||
import asyncio
|
||||
import uuid
|
||||
import re
|
||||
from time import time
|
||||
|
||||
import db
|
||||
|
||||
import get_sq_env
|
||||
|
||||
@@ -106,17 +103,7 @@ async def acat2id(cat_id: str):
|
||||
{
|
||||
"id": obj["id"],
|
||||
"name": obj.get("item_data", {}).get("name", "Unknown"),
|
||||
"number": (
|
||||
None
|
||||
if not obj.get("item_data", {}).get("name", "")
|
||||
else (
|
||||
get_set_number(obj.get("item_data", {}).get("name", ""))
|
||||
if re.findall(r"\d{5}", obj.get("item_data", {}).get("name", ""))
|
||||
else None
|
||||
)
|
||||
),
|
||||
"updated_at": obj.get("updated_at"),
|
||||
"version": obj.get("version"),
|
||||
"number": get_set_number(obj.get("item_data", {}).get("name", ""))
|
||||
}
|
||||
for obj in result_json.get("objects", [])
|
||||
]
|
||||
@@ -125,10 +112,14 @@ async def acat2id(cat_id: str):
|
||||
|
||||
|
||||
async def test() -> None:
|
||||
result = await acat2id(LEGO_CAT_ID) # type: ignore
|
||||
task = asyncio.create_task(acat2id(LEGO_CAT_ID)) # type: ignore
|
||||
await asyncio.sleep(1) # Give the task some time to complete
|
||||
result = await task
|
||||
print("OBJECTS IN LEGO CATEGORY:")
|
||||
for item in result:
|
||||
print(f"ID: {item['id']}, Name: {item['name']}, Updated: {item['updated_at']}")
|
||||
print(f"ID: {item['id']}, Name: {item['name']}")
|
||||
# eturn
|
||||
# pass
|
||||
|
||||
|
||||
"""
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import re
|
||||
from typing import List, Dict, Any, Optional
|
||||
from square import AsyncSquare
|
||||
import get_sq_env
|
||||
|
||||
|
||||
class AsyncSquareClient:
|
||||
"""Asynchronous client for the Square API with focus on catalog operations"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: Optional[str] = None,
|
||||
environment: Optional[str] = None,
|
||||
logger=None,
|
||||
):
|
||||
"""
|
||||
Initialize the AsyncSquareClient.
|
||||
|
||||
Args:
|
||||
token (str, optional): Square API token. If not provided, fetches from environment.
|
||||
environment (str, optional): Square environment (sandbox/production). If not provided, fetches from environment.
|
||||
logger: Optional logger object for logging operations.
|
||||
"""
|
||||
self.token = token or get_sq_env.SQUARE_TOKEN
|
||||
self.environment = environment or get_sq_env.environment
|
||||
self.lego_cat_id = get_sq_env.LEGO_CAT_ID
|
||||
self.client = AsyncSquare(token=self.token, environment=self.environment) # type: ignore
|
||||
self.logger = logger
|
||||
|
||||
@staticmethod
|
||||
def get_set_number(text: str) -> int:
|
||||
"""
|
||||
Extract a five-digit LEGO set number from a string.
|
||||
|
||||
Args:
|
||||
text (str): The text to search for a set number.
|
||||
|
||||
Returns:
|
||||
int: The extracted set number.
|
||||
|
||||
Raises:
|
||||
ValueError: If no five-digit number is found.
|
||||
"""
|
||||
numbers = re.findall(r"\d{5}", text)
|
||||
if not numbers:
|
||||
raise ValueError(f"No five-digit number found in string {text}")
|
||||
return int(numbers[0])
|
||||
|
||||
async def get_item(self, sku: str) -> Dict[str, str]:
|
||||
"""
|
||||
Asynchronously retrieves item details from Square Catalog using a SKU.
|
||||
|
||||
Args:
|
||||
sku (str): The SKU or text filter to search for the item.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the item's name and ID.
|
||||
|
||||
Raises:
|
||||
IndexError: If no items are found matching the SKU.
|
||||
Exception: If there is an error communicating with the Square API.
|
||||
"""
|
||||
if self.logger:
|
||||
self.logger.debug(f"Searching for item with SKU: {sku}")
|
||||
|
||||
result = await self.client.catalog.search_items(text_filter=sku, limit=1)
|
||||
result_json = json.loads(result.model_dump_json())
|
||||
|
||||
try:
|
||||
name = result_json["items"][0]["item_data"]["name"]
|
||||
itm_id = result_json["items"][0]["id"]
|
||||
|
||||
if self.logger:
|
||||
self.logger.info(f"Found item: {name} with ID: {itm_id}")
|
||||
|
||||
return {"name": name, "id": itm_id}
|
||||
except (KeyError, IndexError) as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error retrieving item with SKU {sku}: {str(e)}")
|
||||
raise
|
||||
|
||||
async def update_item(self, lego_set: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Asynchronously updates an item in the Square catalog based on provided LEGO set information.
|
||||
|
||||
Args:
|
||||
lego_set (dict): A dictionary containing LEGO set details including:
|
||||
- sets: List containing set information
|
||||
- pw_name: PriceWise system name
|
||||
- set_adjustredprice: Adjusted retail price
|
||||
|
||||
Returns:
|
||||
bool: True if update was successful.
|
||||
|
||||
Raises:
|
||||
KeyError: If required keys are missing from input dictionary.
|
||||
Exception: For Square API communication errors.
|
||||
"""
|
||||
if self.logger:
|
||||
self.logger.debug(
|
||||
f"Updating item with set info: {lego_set['sets'][0]['number']}"
|
||||
)
|
||||
|
||||
try:
|
||||
set_name = lego_set["sets"][0]["name"]
|
||||
set_number = lego_set["sets"][0]["number"]
|
||||
set_upc = lego_set["sets"][0]["barcode"]["UPC"]
|
||||
set_image = lego_set["sets"][0]["image"]["thumbnailURL"]
|
||||
pw_name = lego_set["pw_name"]
|
||||
set_adjustedprice = lego_set["set_adjustredprice"]
|
||||
|
||||
result = await self.client.catalog.search_items(
|
||||
text_filter=set_number, limit=1
|
||||
)
|
||||
itm_id = json.loads(result.model_dump_json())["items"][0]["id"]
|
||||
|
||||
if self.logger:
|
||||
self.logger.info(f"Successfully updated item {set_number}")
|
||||
|
||||
# TODO: Implement actual update logic here
|
||||
# await self.client.catalog.object.upsert()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error updating item: {str(e)}")
|
||||
raise
|
||||
|
||||
async def get_category_items(
|
||||
self, cat_id: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Return a list of items within the specified category ID.
|
||||
|
||||
Args:
|
||||
cat_id (str, optional): The category ID to search for items.
|
||||
If not provided, uses the default LEGO category ID.
|
||||
|
||||
Returns:
|
||||
list[dict]: A list of dictionaries, each containing:
|
||||
- 'id': The item's ID
|
||||
- 'name': The item's name
|
||||
- 'number': The LEGO set number
|
||||
"""
|
||||
category_id = cat_id or self.lego_cat_id
|
||||
|
||||
if self.logger:
|
||||
self.logger.debug(f"Fetching items for category: {category_id}")
|
||||
|
||||
try:
|
||||
# Create a properly structured query for the Square API
|
||||
search_request = {
|
||||
"object_types": ["ITEM"],
|
||||
"query": {
|
||||
"exact_query": {
|
||||
"attribute_name": "category_id",
|
||||
"attribute_value": category_id,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
result = await self.client.catalog.search(**search_request)
|
||||
result_json = json.loads(result.model_dump_json())
|
||||
|
||||
items = [
|
||||
{
|
||||
"id": obj["id"],
|
||||
"name": obj.get("item_data", {}).get("name", "Unknown"),
|
||||
"number": self.get_set_number(
|
||||
obj.get("item_data", {}).get("name", "")
|
||||
),
|
||||
}
|
||||
for obj in result_json.get("objects", [])
|
||||
]
|
||||
|
||||
if self.logger:
|
||||
self.logger.info(f"Found {len(items)} items in category {category_id}")
|
||||
|
||||
return items
|
||||
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.error(f"Error fetching category items: {str(e)}")
|
||||
raise
|
||||
|
||||
|
||||
# Provide acat2id as a module-level function for backward compatibility
|
||||
async def acat2id(cat_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Backward compatibility function for getting category items.
|
||||
|
||||
Args:
|
||||
cat_id (str): The category ID to search for items.
|
||||
|
||||
Returns:
|
||||
list[dict]: A list of dictionaries containing item information.
|
||||
"""
|
||||
client = AsyncSquareClient()
|
||||
return await client.get_category_items(cat_id)
|
||||
|
||||
|
||||
async def test() -> None:
|
||||
"""Test function to demonstrate class usage"""
|
||||
client = AsyncSquareClient()
|
||||
print(f"@[DEBUG] ENV:\t{client.environment}")
|
||||
print(f"@[DEBUG] CATID:\t{client.lego_cat_id}")
|
||||
|
||||
result = await client.get_category_items()
|
||||
print("\nOBJECTS IN LEGO CATEGORY:")
|
||||
for item in result:
|
||||
print(f"ID: {item['id']}, Name: {item['name']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test())
|
||||
+3
-8
@@ -14,9 +14,9 @@ debug = False # Set to True to enable debug output
|
||||
|
||||
def login(username, password):
|
||||
url = "https://brickset.com/api/v3.asmx/login"
|
||||
headers = {"UserAgent": "studspider/0.1"}
|
||||
headers = {"UserAgent": "Edderkop/0.1"}
|
||||
params = {
|
||||
"UserAgent": "studspider/0.1",
|
||||
"UserAgent": "Edderkop/0.1",
|
||||
"username": username,
|
||||
"password": password,
|
||||
"apiKey": BRICKSET_API_KEY,
|
||||
@@ -33,11 +33,6 @@ def get_set(set_number: int) -> dict:
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if not data["sets"] and data["message"] == "API limit exceeded":
|
||||
raise Exception("API limit exceeded. Please try again later.")
|
||||
return False
|
||||
|
||||
print(f"[@DEBUG]\t{data}")
|
||||
our_set = {
|
||||
"set_num": data["sets"][0]["number"],
|
||||
"set_name": data["sets"][0]["name"],
|
||||
@@ -48,7 +43,7 @@ def get_set(set_number: int) -> dict:
|
||||
}
|
||||
# add PW 10% b4 tax
|
||||
our_set.update(
|
||||
{"set_adjustredprice": our_set["set_retailprice"] + ((10 / 100) * our_set["set_retailprice"])}
|
||||
{"set_adjustredprice": our_set + ((10 / 100) * our_set["set_retailprice"])}
|
||||
)
|
||||
our_set["pw_name"] = (
|
||||
f"{our_set['set_num']} - {our_set["set_theme"].upper()} {our_set['set_name']}"
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
import aiosqlite
|
||||
from time import time
|
||||
|
||||
time = int(time()) # we dont need no stinkin floats
|
||||
|
||||
# db = await aiosqlite.connect("studspider.db") # type: ignore
|
||||
|
||||
|
||||
async def close(db):
|
||||
await db.close()
|
||||
|
||||
|
||||
async def fetch(table, sqobj):
|
||||
"""
|
||||
Asynchronously fetch a record from the REVIEW table by sq_object.
|
||||
|
||||
Args:
|
||||
sqobj (str): The sq_object identifier to search for.
|
||||
Returns:
|
||||
dict or None: The fetched record as a dictionary, or None if not found.
|
||||
"""
|
||||
async with aiosqlite.connect("studspider.db") as db: # type: ignore
|
||||
async with db.execute(
|
||||
f"SELECT * FROM {table} WHERE sq_object = {sqobj}"
|
||||
) as cursor:
|
||||
async for row in cursor:
|
||||
return row
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def upsert(table: str, data: dict, unique_key: str) -> bool:
|
||||
"""
|
||||
Asynchronously insert or update a record in the specified table.
|
||||
|
||||
Args:
|
||||
table (str): The name of the table to upsert into.
|
||||
data (dict): A dictionary of column-value pairs to insert or update.
|
||||
unique_key (str): The column name that serves as the unique key for upsert.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
print(f"Upserting into {table}: {data}")
|
||||
|
||||
columns = ", ".join(data.keys())
|
||||
placeholders = ", ".join("?" * len(data))
|
||||
updates = ", ".join(
|
||||
f"{col}=excluded.{col}" for col in data.keys() if col != unique_key
|
||||
)
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO {table} ({columns})
|
||||
VALUES ({placeholders})
|
||||
ON CONFLICT({unique_key}) DO UPDATE SET
|
||||
{updates};
|
||||
"""
|
||||
async with aiosqlite.connect("studspider.db") as db: # type: ignore
|
||||
async with db.execute(sql, tuple(data.values())) as cursor:
|
||||
await db.commit()
|
||||
return True
|
||||
@@ -6,21 +6,21 @@ import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Get the absolute path to studspider.db in the project root
|
||||
# Get the absolute path to edderkop.db in the project root
|
||||
DEFAULT_DB_PATH = os.path.join(
|
||||
os.path.dirname(os.path.dirname(__file__)), "studspider.db"
|
||||
os.path.dirname(os.path.dirname(__file__)), "edderkop.db"
|
||||
)
|
||||
|
||||
if not os.path.exists(DEFAULT_DB_PATH):
|
||||
raise FileNotFoundError(f"Database file not found at {DEFAULT_DB_PATH}")
|
||||
print('Please run "src/scripts/studspider.SQL" to setup the database.')
|
||||
print('Please run "src/scripts/EDDERKOP.SQL" to setup the database.')
|
||||
os.exit(0)
|
||||
|
||||
|
||||
class AsyncSchedulerDB:
|
||||
"""
|
||||
Asynchronous SQLite database manager for APScheduler.
|
||||
Handles job storage, retrieval, and database operations using the main studspider.db database.
|
||||
Handles job storage, retrieval, and database operations using the main edderkop.db database.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str = DEFAULT_DB_PATH):
|
||||
@@ -29,7 +29,7 @@ class AsyncSchedulerDB:
|
||||
|
||||
Args:
|
||||
db_path (str, optional): Path to the SQLite database file.
|
||||
Defaults to studspider.db in project root.
|
||||
Defaults to edderkop.db in project root.
|
||||
"""
|
||||
self.db_path = db_path
|
||||
self._initialized = False
|
||||
+1
-1
@@ -15,7 +15,7 @@ class BricksetAPI:
|
||||
self.password = os.getenv("BRICKSET_PASS")
|
||||
self.api_key = os.getenv("BRICKSET_API_KEY")
|
||||
self.debug = debug
|
||||
self.headers = {"UserAgent": "studspider/0.1"}
|
||||
self.headers = {"UserAgent": "Edderkop/0.1"}
|
||||
self.user_hash = None
|
||||
|
||||
def login(self) -> Dict[str, Any]:
|
||||
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from . import get_sq_env
|
||||
import aiosqlite
|
||||
from time import time
|
||||
|
||||
from . import asquare
|
||||
from . import db
|
||||
from . import abricksetapi
|
||||
from . import throttler
|
||||
Throttler = throttler.GlobalThrottler
|
||||
|
||||
|
||||
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
|
||||
|
||||
# big glue boy
|
||||
async def do_review() -> bool:
|
||||
result = await asquare.acat2id(LEGO_CAT_ID) # type: ignore
|
||||
for item in result:
|
||||
if not re.match(r".*\d{5}.*", item["name"]):
|
||||
missing = []
|
||||
print(f"{item} is missing its set number!")
|
||||
#print(f"[@DEBUG] Upserting {item['id']} into REVIEW table")
|
||||
item_sanitized = {
|
||||
"sq_object": item["id"],
|
||||
"set_name": item["name"],
|
||||
"set_number": None,
|
||||
"time_added": time
|
||||
}
|
||||
missing.append(item_sanitized)
|
||||
|
||||
for item in missing:
|
||||
print(item)
|
||||
# sq_item = await db.fetch(item["id"])
|
||||
if item["id"] != sq_item["sq_object"]:
|
||||
print(f"[@DEBUG] {item['id']} is missing)")
|
||||
#await db.upsert("MISSING", item_sanitized, "sq_object")
|
||||
# #print(f"[@DEBUG] Upserted {item['id']} into REVIEW table")
|
||||
#print(f"[@DEBUG] {item['id']} already in REVIEW table")
|
||||
return True
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import json
|
||||
|
||||
from . import get_sq_env
|
||||
import get_sq_env
|
||||
|
||||
SQUARE_TOKEN = get_sq_env.SQUARE_TOKEN
|
||||
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE "REVIEW" (
|
||||
"sq_object" TEXT NOT NULL UNIQUE,
|
||||
"time_added" INTEGER,
|
||||
"set_number" INTEGER,
|
||||
"time_modified" INTEGER,
|
||||
PRIMARY KEY("sq_object")
|
||||
)
|
||||
@@ -1,15 +0,0 @@
|
||||
CREATE TABLE "REVIEW" (
|
||||
"sq_object" TEXT NOT NULL UNIQUE,
|
||||
"price" INTEGER NOT NULL,
|
||||
"date_modified" INTEGER,
|
||||
PRIMARY KEY("sq_object")
|
||||
)
|
||||
|
||||
CREATE TABLE "MISSING" (
|
||||
"sq_object" TEXT NOT NULL UNIQUE,
|
||||
"time_added" INTEGER,
|
||||
"set_number" INTEGER,
|
||||
"time_modified" INTEGER,
|
||||
"set_name" TEXT,
|
||||
PRIMARY KEY("sq_object")
|
||||
)
|
||||
+2
-2
@@ -74,7 +74,7 @@ async def handle_post(request):
|
||||
return True
|
||||
else:
|
||||
return aiohttp.web.Response(status=400, text="400 Bad Request.")
|
||||
result = {"message": f"studspider/0.1 received command: {reqdata}"}
|
||||
result = {"message": f"Edderkop/0.1 received command: {reqdata}"}
|
||||
return aiohttp.web.json_response(status=200, data=result)
|
||||
|
||||
|
||||
@@ -108,6 +108,6 @@ async def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(generate_toilet_banner("studspider v0.1"))
|
||||
print(generate_toilet_banner("Edderkop v0.1"))
|
||||
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
|
||||
aiohttp.web.run_app(app, port=7432)
|
||||
@@ -0,0 +1,6 @@
|
||||
import src.libsquare as sq
|
||||
import src.bricksetapi as lego
|
||||
|
||||
oursq = sq.get_itm("71417")
|
||||
ourlego = lego.get_set(71417)
|
||||
print(f"{oursq}\n\n{ourlego}\n\n")
|
||||
+21
-138
@@ -1,78 +1,52 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from aiohttp import web
|
||||
from base64 import b64decode
|
||||
from typing import Dict, Any
|
||||
import uuid
|
||||
|
||||
from time import time
|
||||
import dotenv
|
||||
import os
|
||||
|
||||
import re
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
print("Creating log directory if it doesn't exist...")
|
||||
os.makedirs("./logs/", exist_ok=True)
|
||||
Path("./logs/").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
import banner
|
||||
import get_sq_env
|
||||
from time import time
|
||||
|
||||
time = int(time())
|
||||
import asquare as sq
|
||||
|
||||
# from asquareclass import AsyncSquareClient # type: ignore
|
||||
from abricksetapi import AsyncBricksetAPI
|
||||
|
||||
print("Loading environment variables...")
|
||||
|
||||
dotenv.load_dotenv()
|
||||
SQUARE_LEGO_CAT_ID = os.environ.get("SQUARE_LEGO_CAT_ID")
|
||||
|
||||
|
||||
from asyncio_throttle.throttler import Throttler
|
||||
|
||||
# set up logger
|
||||
print("Setting up logger...")
|
||||
logger = logging.getLogger("studspider")
|
||||
logger = logging.getLogger("edderkop")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
handler = RotatingFileHandler("./logs/studspider.log", maxBytes=125000, backupCount=5)
|
||||
handler = RotatingFileHandler("./logs/", maxBytes=1024, backupCount=5)
|
||||
logger.addHandler(handler)
|
||||
|
||||
lego = AsyncBricksetAPI(debug=True, logger=logger)
|
||||
# sq = AsyncSquareClient(logger=logger)
|
||||
|
||||
# logging.basicConfig(
|
||||
# filename="./logs/studspider.log", encoding="utf-8", level=logging.DEBUG
|
||||
# filename="./logs/edderkop.log", encoding="utf-8", level=logging.DEBUG
|
||||
# )
|
||||
print("Setting up rate limiter...")
|
||||
print("Doing RE things...")
|
||||
|
||||
UA = re.compile(r"^kopling\/?", re.IGNORECASE)
|
||||
|
||||
|
||||
async def run_update():
|
||||
set_list = await sq.acat2id(SQUARE_LEGO_CAT_ID) # type: ignore
|
||||
# print(f"[@DEBUG]\t {set_list}") # type: ignore
|
||||
for l_set in set_list:
|
||||
# print(f"[@DEBUG]{l_set}")
|
||||
try:
|
||||
l_setinfo = await lego.throttled_get_set(l_set["number"])
|
||||
print(f"[@DEBUG] Found {l_setinfo["pw_name"]}")
|
||||
except Exception as e:
|
||||
print(f"Error fetching set\t: {l_set}: {str(e)}")
|
||||
continue
|
||||
|
||||
# print(f"[@DEBUG]{l_setinfo}")
|
||||
# await sq.update_itm(l_set, l_setinfo)
|
||||
"""for l_set in set_list:
|
||||
print(f"[@DEBUG]{l_set}")
|
||||
try:
|
||||
l_setinfo = await lego.throttled_get_set(l_set["number"])
|
||||
except Exception as e:
|
||||
print(f"Error fetching set\t: {l_set}: {str(e)}")
|
||||
continue
|
||||
|
||||
print(f"[@DEBUG]{l_setinfo}")"""
|
||||
print(f"[@DEBUG]{l_setinfo}")
|
||||
# await sq.update_itm(l_set, l_setinfo)
|
||||
|
||||
|
||||
@@ -84,53 +58,14 @@ def valid_uuid(val):
|
||||
return False
|
||||
|
||||
|
||||
async def hook_handler(request: web.Request) -> web.Response:
|
||||
"""Handle incoming JSON webhook POSTs on /hook endpoint"""
|
||||
try:
|
||||
# Verify request method
|
||||
if request.method != "POST":
|
||||
logger.error(
|
||||
f"{time}\tInvalid request method to /hook from %s", request.remote
|
||||
)
|
||||
return web.Response(status=405, text="Method not allowed. Use POST.")
|
||||
|
||||
# Get JSON data from request
|
||||
try:
|
||||
data = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
logger.error("Invalid JSON to /hook from %s", request.remote)
|
||||
return web.Response(status=400, text="Invalid JSON in request body")
|
||||
|
||||
# Log the received webhook
|
||||
print(f"[@DEBUG] Webhook received at {time} from {request.remote}")
|
||||
logger.info(
|
||||
f"{time}\tWebhook received from {request.remote}: {json.dumps(data)}"
|
||||
)
|
||||
|
||||
# Process the webhook data here
|
||||
# TODO: Add your webhook processing logic
|
||||
|
||||
response_data = {
|
||||
"status": "success",
|
||||
"message": "Webhook received",
|
||||
"timestamp": time,
|
||||
"data_received": data,
|
||||
}
|
||||
|
||||
return web.json_response(response_data)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Webhook handler error: {str(e)}")
|
||||
print(f"Webhook handler error: {str(e)}")
|
||||
return web.Response(status=500, text="Internal server error")
|
||||
|
||||
|
||||
async def request_handler(request: web.Request) -> web.Response:
|
||||
"""Handle incoming POST requests with command processing"""
|
||||
try:
|
||||
# Verify request method
|
||||
if request.method != "POST":
|
||||
logger.error(f"{time}\tInvalid request method from %s at ", request.remote)
|
||||
logger.error(
|
||||
f"{time()}\tInvalid request method from %s at ", request.remote
|
||||
)
|
||||
return web.Response(status=405, text="Method not allowed. Use POST.")
|
||||
|
||||
# Get JSON data from request
|
||||
@@ -158,17 +93,19 @@ async def request_handler(request: web.Request) -> web.Response:
|
||||
try:
|
||||
decoded_cmd = b64decode(data["CMD"]).decode("utf-8")
|
||||
print(
|
||||
f"""At: {time}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
|
||||
f"""At: {time()}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
|
||||
)
|
||||
logging.info(
|
||||
f"{time}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
|
||||
f"{time()}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
|
||||
)
|
||||
|
||||
# Process the command
|
||||
if decoded_cmd == "RUNUPDATE":
|
||||
response_data = {"status": "success", "message": "Update started"}
|
||||
# Add your command handling logic here
|
||||
await run_update()
|
||||
response_data = {"status": "success", "message": "Update started"}
|
||||
logging.info(f"{time()}\tUpdate command processed successfully")
|
||||
|
||||
return web.json_response(response_data)
|
||||
else:
|
||||
return web.Response(status=400, text=f"Unknown command: {decoded_cmd}")
|
||||
@@ -182,64 +119,10 @@ async def request_handler(request: web.Request) -> web.Response:
|
||||
return web.Response(status=500, text="Internal server error")
|
||||
|
||||
|
||||
print("Starting web server...")
|
||||
os.system("cls" if os.name == "nt" else "clear")
|
||||
app = web.Application()
|
||||
app.router.add_post("/api", request_handler)
|
||||
app.router.add_post("/hook", hook_handler)
|
||||
|
||||
parser = argparse.ArgumentParser(description="studspider")
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
help="IP address to listen on. default: localhost",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=7432, help="Port to listen on. default: 7432"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hook-port",
|
||||
type=int,
|
||||
default=8080,
|
||||
help="Optional separate port for webhook endpoint. default: 8080",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
app.router.add_post("/", request_handler)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(banner.generate("studspider v0.1"))
|
||||
print(banner.generate("Edderkop v0.1"))
|
||||
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
|
||||
|
||||
# If hook-port is specified, run a separate server for webhooks
|
||||
if args.hook_port:
|
||||
print(f"Starting main server on {args.host}:{args.port}")
|
||||
print(f"Starting webhook server on {args.host}:{args.hook_port}")
|
||||
|
||||
# Create separate app for webhook
|
||||
hook_app = web.Application()
|
||||
hook_app.router.add_post("/hook", hook_handler)
|
||||
|
||||
# Run both servers concurrently
|
||||
async def run_servers():
|
||||
runner1 = web.AppRunner(app)
|
||||
await runner1.setup()
|
||||
site1 = web.TCPSite(runner1, args.host, args.port)
|
||||
|
||||
runner2 = web.AppRunner(hook_app)
|
||||
await runner2.setup()
|
||||
site2 = web.TCPSite(runner2, args.host, args.hook_port)
|
||||
|
||||
await site1.start()
|
||||
await site2.start()
|
||||
|
||||
print(f"Main server running on http://{args.host}:{args.port}")
|
||||
print(f"Webhook server running on http://{args.host}:{args.hook_port}/hook")
|
||||
|
||||
# Keep running
|
||||
await asyncio.Event().wait()
|
||||
|
||||
asyncio.run(run_servers())
|
||||
else:
|
||||
print(f"Server running on {args.host}:{args.port}")
|
||||
print(f"Webhook endpoint: http://{args.host}:{args.port}/hook")
|
||||
web.run_app(app, host=args.host, port=args.port)
|
||||
web.run_app(app, host="localhost", port=7432)
|
||||
@@ -1,33 +0,0 @@
|
||||
import asyncio
|
||||
from asyncio_throttle import Throttler as AsyncThrottler
|
||||
|
||||
class GlobalThrottler:
|
||||
_instance = None
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
def __init__(self, rate_limit: int = 500, period: int = 60) -> None:
|
||||
if not hasattr(self, '_initialized'):
|
||||
self.rate_limit = rate_limit
|
||||
self.period = period
|
||||
self.throttler = AsyncThrottler(rate_limit=self.rate_limit, period=self.period)
|
||||
self._initialized = True
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls, rate_limit: int = 500, period: int = 60) -> 'GlobalThrottler':
|
||||
async with cls._lock:
|
||||
if not cls._instance:
|
||||
cls._instance = cls(rate_limit, period)
|
||||
return cls._instance
|
||||
|
||||
async def acquire(self):
|
||||
await self.throttler.acquire()
|
||||
|
||||
def release(self):
|
||||
self.throttler.release()
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.acquire()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
self.release()
|
||||
File renamed without changes.
@@ -1,31 +0,0 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from pprint import pprint
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import socket
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
|
||||
import src.libsquare as sq
|
||||
import src.bricksetapi as lego
|
||||
|
||||
print("=" * 80)
|
||||
result = sock.connect_ex(('127.0.0.1', 7432))
|
||||
if result == 0:
|
||||
print("PORT 7432 IS IN USE")
|
||||
else:
|
||||
print("PORT 7432 IS AVAILABLE")
|
||||
sock.close()
|
||||
|
||||
print("=" * 80)
|
||||
print("SQUARE OBJECT FOR SET 71417:")
|
||||
print("=" * 80)
|
||||
oursq = sq.get_itm("71417")
|
||||
pprint(oursq, width=120, depth=None)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("LEGO SET INFORMATION FOR SET 71417:")
|
||||
print("=" * 80)
|
||||
ourlego = lego.get_set(71417)
|
||||
print("=" * 80)
|
||||
pprint(ourlego, width=120, depth=None)
|
||||
Reference in new issue
Block a user