10 Commits
Author SHA1 Message Date
C d32c17a5da add stuff todo for fixing database 2025-10-09 17:15:12 -07:00
C 729746b0bd uv refresh 2025-10-09 17:14:59 -07:00
C 152bfb316b add verbosity to startup 2025-10-09 17:14:53 -07:00
C 369ccdbfc0 testing subclassed asquare 2025-10-09 17:14:37 -07:00
C a419dd498f handle no numbers in set name 2025-10-09 17:13:56 -07:00
C b4b0ccdac6 added rotating logs! epicsauce 2025-10-09 13:33:59 -07:00
C 32664c3a10 added optional logging to brickset 2025-10-09 13:33:51 -07:00
C 41ce94c85c Auto stash before rebase of "uv" 2025-10-09 12:55:38 -07:00
ph0sgene 397eddf299 rm 314t 2025-10-09 12:54:32 -07:00
C 8affb90308 refactor: Migrate build system to Hatch and integrate UV 2025-10-09 12:49:37 -07:00
25 changed files with 74 additions and 1562 deletions

No files matched your search

+2 -11
View File
@@ -8,6 +8,8 @@ __pycache__/
*.db
*.sqlite3
*.log
poetry.lock
uv.lock
bin/build/
.zip
snippet.py
@@ -24,14 +26,3 @@ doc/pw-inventory/square_inventory_export_2025-06-18-0137.xlsx
square-src.code-workspace
doc/pw-inventory/
studspider.db-journal
.venv
*.db
*.py[cod]
.web
.states
assets/external/
__pycache__/
win/
.github/
-21
View File
@@ -1,21 +0,0 @@
FROM python:3.13-alpine
# Install uv
RUN pip install uv
# Set working directory
WORKDIR /app
# Copy project files
COPY pyproject.toml ./
COPY README.md ./
# Install dependencies using uv without venv
RUN uv pip compile pyproject.toml -o requirements.txt && \
uv pip install --system -r requirements.txt
# Copy the rest of the application
COPY . .
# Default command
CMD ["python", "tests/test.py"]
+4 -35
View File
@@ -1,43 +1,12 @@
# studspider
# Edderkop
![Static Badge](https://img.shields.io/badge/python-3.13.7-blue?logo=python) ![Static Badge](https://img.shields.io/badge/code_style-black-black) ![Static Badge](https://img.shields.io/badge/license-MPL-orange)
environment: poetry
code style: black
is a tool used to talk to brickset.com and a square inventory to pull lego set information for entering in the square db
## Run me
Clone the project
```bash
git clone http://git.traps.moe/ph0sgene/studspider.git
git checkout uv
```
Go to the project directory
```bash
cd studspider
```
Install dependencies
```bash
uv sync && uv lock
```
Start the server
```bash
uvx run studspider.py
```
## Tech
built with
- aiohttp
- squareup
- apscheduler
- aiosqlite
+3 -3
View File
@@ -1,12 +1,12 @@
import src.test_server
class Studspider:
class edderkop:
def __init__(self):
self.name = "Studspider"
self.name = "Edderkop"
self.version = "0.0.1"
self.description = """
studspider is an application designed to interact with the Brickset API and directly with Square.
Edderkop is an application designed to interact with the Brickset API and directly with Square.
It allows users to search for items, manage collections, and perform various operations related to bricksets.
"""
Binary file not shown.

Before

Width:  |  Height:  |  Size: 68 KiB

+1 -3
View File
@@ -1,5 +1,5 @@
[project]
name = "studpider"
name = "edderkop"
version = "0.1.0"
description = ""
authors = [{ name = "CF", email = "ph0sgene@pm.me" }]
@@ -16,8 +16,6 @@ dependencies = [
"pytest (>=8.4.2,<9.0.0)",
"apscheduler (>=3.11.0,<4.0.0)",
"aiosqlite (>=0.21.0,<0.22.0)",
"bs4>=0.0.2",
"reflex>=0.8.14.post1",
]
[build-system]
+9 -9
View File
@@ -1,19 +1,19 @@
from setuptools import setup
setup(
name="studspider",
version="0.1",
description="Bricket + Square integration application",
author="CF",
author_email="chris-forst@pm.me",
packages=["src"],
package_dir={"": "src"},
name='Edderkop',
version='0.1',
description='Bricket + Square integration application',
author='CF',
author_email='chris-forst@pm.me',
packages=['src'],
package_dir={'': 'src'},
install_requires=[
# List your project dependencies here
],
entry_points={
"console_scripts": [
"main=main:main", # Adjust this line based on your main function
'console_scripts': [
'main=main:main', # Adjust this line based on your main function
],
},
)
View File
Whitespace-only changes.
+3 -6
View File
@@ -9,9 +9,7 @@ import aiohttp
import json
import dotenv
from typing import Dict, Any
import throttler
Throttler = throttler.GlobalThrottler
from asyncio_throttle.throttler import Throttler
dotenv.load_dotenv()
@@ -24,7 +22,7 @@ class AsyncBricksetAPI:
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
self.debug = debug
self.headers = {"UserAgent": "studspider/0.1"}
self.headers = {"UserAgent": "Edderkop/0.1"}
self.throttler = Throttler(rate_limit=500, period=60)
self.user_hash = None
self.logger = logger
@@ -55,7 +53,7 @@ class AsyncBricksetAPI:
self.logger.info("Successfully authenticated with Brickset API")
return result
async def get_set(self, set_number: int) -> Dict[str, Any]:
async def get_set(self, set_number: str) -> Dict[str, Any]:
"""
Fetch LEGO set information asynchronously
@@ -74,7 +72,6 @@ class AsyncBricksetAPI:
"userHash": self.user_hash,
"params": json.dumps({"setNumber": f"{set_number}-1"}),
}
print(f"[@DEBUG] {params}...")
async with aiohttp.ClientSession() as session:
async with session.get(
+11 -16
View File
@@ -4,9 +4,6 @@ import json
import asyncio
import uuid
import re
from time import time
import db
import get_sq_env
@@ -106,17 +103,11 @@ async def acat2id(cat_id: str):
{
"id": obj["id"],
"name": obj.get("item_data", {}).get("name", "Unknown"),
"number": (
None
if not obj.get("item_data", {}).get("name", "")
else (
get_set_number(obj.get("item_data", {}).get("name", ""))
if re.findall(r"\d{5}", obj.get("item_data", {}).get("name", ""))
else None
)
),
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
"number": None if not obj.get("item_data", {}).get("name", "") else (
get_set_number(obj.get("item_data", {}).get("name", ""))
if re.findall(r"\d{5}", obj.get("item_data", {}).get("name", ""))
else None
)
}
for obj in result_json.get("objects", [])
]
@@ -125,10 +116,14 @@ async def acat2id(cat_id: str):
async def test() -> None:
result = await acat2id(LEGO_CAT_ID) # type: ignore
task = asyncio.create_task(acat2id(LEGO_CAT_ID)) # type: ignore
await asyncio.sleep(1) # Give the task some time to complete
result = await task
print("OBJECTS IN LEGO CATEGORY:")
for item in result:
print(f"ID: {item['id']}, Name: {item['name']}, Updated: {item['updated_at']}")
print(f"ID: {item['id']}, Name: {item['name']}")
# eturn
# pass
"""
+3 -8
View File
@@ -14,9 +14,9 @@ debug = False # Set to True to enable debug output
def login(username, password):
url = "https://brickset.com/api/v3.asmx/login"
headers = {"UserAgent": "studspider/0.1"}
headers = {"UserAgent": "Edderkop/0.1"}
params = {
"UserAgent": "studspider/0.1",
"UserAgent": "Edderkop/0.1",
"username": username,
"password": password,
"apiKey": BRICKSET_API_KEY,
@@ -33,11 +33,6 @@ def get_set(set_number: int) -> dict:
response = requests.get(url)
if response.status_code == 200:
data = response.json()
if not data["sets"] and data["message"] == "API limit exceeded":
raise Exception("API limit exceeded. Please try again later.")
return False
print(f"[@DEBUG]\t{data}")
our_set = {
"set_num": data["sets"][0]["number"],
"set_name": data["sets"][0]["name"],
@@ -48,7 +43,7 @@ def get_set(set_number: int) -> dict:
}
# add PW 10% b4 tax
our_set.update(
{"set_adjustredprice": our_set["set_retailprice"] + ((10 / 100) * our_set["set_retailprice"])}
{"set_adjustredprice": our_set + ((10 / 100) * our_set["set_retailprice"])}
)
our_set["pw_name"] = (
f"{our_set['set_num']} - {our_set["set_theme"].upper()} {our_set['set_name']}"
-61
View File
@@ -1,61 +0,0 @@
import aiosqlite
from time import time
time = int(time()) # we dont need no stinkin floats
# db = await aiosqlite.connect("studspider.db") # type: ignore
async def close(db):
await db.close()
async def fetch(table, sqobj):
"""
Asynchronously fetch a record from the REVIEW table by sq_object.
Args:
sqobj (str): The sq_object identifier to search for.
Returns:
dict or None: The fetched record as a dictionary, or None if not found.
"""
async with aiosqlite.connect("studspider.db") as db: # type: ignore
async with db.execute(
f"SELECT * FROM {table} WHERE sq_object = {sqobj}"
) as cursor:
async for row in cursor:
return row
return None
async def upsert(table: str, data: dict, unique_key: str) -> bool:
"""
Asynchronously insert or update a record in the specified table.
Args:
table (str): The name of the table to upsert into.
data (dict): A dictionary of column-value pairs to insert or update.
unique_key (str): The column name that serves as the unique key for upsert.
Returns:
None
"""
print(f"Upserting into {table}: {data}")
columns = ", ".join(data.keys())
placeholders = ", ".join("?" * len(data))
updates = ", ".join(
f"{col}=excluded.{col}" for col in data.keys() if col != unique_key
)
sql = f"""
INSERT INTO {table} ({columns})
VALUES ({placeholders})
ON CONFLICT({unique_key}) DO UPDATE SET
{updates};
"""
async with aiosqlite.connect("studspider.db") as db: # type: ignore
async with db.execute(sql, tuple(data.values())) as cursor:
await db.commit()
return True
+5 -5
View File
@@ -6,21 +6,21 @@ import os
logger = logging.getLogger(__name__)
# Get the absolute path to studspider.db in the project root
# Get the absolute path to edderkop.db in the project root
DEFAULT_DB_PATH = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "studspider.db"
os.path.dirname(os.path.dirname(__file__)), "edderkop.db"
)
if not os.path.exists(DEFAULT_DB_PATH):
raise FileNotFoundError(f"Database file not found at {DEFAULT_DB_PATH}")
print('Please run "src/scripts/studspider.SQL" to setup the database.')
print('Please run "src/scripts/EDDERKOP.SQL" to setup the database.')
os.exit(0)
class AsyncSchedulerDB:
"""
Asynchronous SQLite database manager for APScheduler.
Handles job storage, retrieval, and database operations using the main studspider.db database.
Handles job storage, retrieval, and database operations using the main edderkop.db database.
"""
def __init__(self, db_path: str = DEFAULT_DB_PATH):
@@ -29,7 +29,7 @@ class AsyncSchedulerDB:
Args:
db_path (str, optional): Path to the SQLite database file.
Defaults to studspider.db in project root.
Defaults to edderkop.db in project root.
"""
self.db_path = db_path
self._initialized = False
+1 -1
View File
@@ -15,7 +15,7 @@ class BricksetAPI:
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
self.debug = debug
self.headers = {"UserAgent": "studspider/0.1"}
self.headers = {"UserAgent": "Edderkop/0.1"}
self.user_hash = None
def login(self) -> Dict[str, Any]:
-41
View File
@@ -1,41 +0,0 @@
import asyncio
import json
import re
from . import get_sq_env
import aiosqlite
from time import time
from . import asquare
from . import db
from . import abricksetapi
from . import throttler
Throttler = throttler.GlobalThrottler
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
# big glue boy
async def do_review() -> bool:
result = await asquare.acat2id(LEGO_CAT_ID) # type: ignore
for item in result:
if not re.match(r".*\d{5}.*", item["name"]):
missing = []
print(f"{item} is missing its set number!")
#print(f"[@DEBUG] Upserting {item['id']} into REVIEW table")
item_sanitized = {
"sq_object": item["id"],
"set_name": item["name"],
"set_number": None,
"time_added": time
}
missing.append(item_sanitized)
for item in missing:
print(item)
# sq_item = await db.fetch(item["id"])
if item["id"] != sq_item["sq_object"]:
print(f"[@DEBUG] {item['id']} is missing)")
#await db.upsert("MISSING", item_sanitized, "sq_object")
# #print(f"[@DEBUG] Upserted {item['id']} into REVIEW table")
#print(f"[@DEBUG] {item['id']} already in REVIEW table")
return True
+1 -1
View File
@@ -1,6 +1,6 @@
import json
from . import get_sq_env
import get_sq_env
SQUARE_TOKEN = get_sq_env.SQUARE_TOKEN
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
+7
View File
@@ -0,0 +1,7 @@
CREATE TABLE "REVIEW" (
"sq_object" TEXT NOT NULL UNIQUE,
"time_added" INTEGER,
"set_number" INTEGER,
"time_modified" INTEGER,
PRIMARY KEY("sq_object")
)
-15
View File
@@ -1,15 +0,0 @@
CREATE TABLE "REVIEW" (
"sq_object" TEXT NOT NULL UNIQUE,
"price" INTEGER NOT NULL,
"date_modified" INTEGER,
PRIMARY KEY("sq_object")
)
CREATE TABLE "MISSING" (
"sq_object" TEXT NOT NULL UNIQUE,
"time_added" INTEGER,
"set_number" INTEGER,
"time_modified" INTEGER,
"set_name" TEXT,
PRIMARY KEY("sq_object")
)
+2 -2
View File
@@ -74,7 +74,7 @@ async def handle_post(request):
return True
else:
return aiohttp.web.Response(status=400, text="400 Bad Request.")
result = {"message": f"studspider/0.1 received command: {reqdata}"}
result = {"message": f"Edderkop/0.1 received command: {reqdata}"}
return aiohttp.web.json_response(status=200, data=result)
@@ -108,6 +108,6 @@ async def main():
if __name__ == "__main__":
print(generate_toilet_banner("studspider v0.1"))
print(generate_toilet_banner("Edderkop v0.1"))
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
aiohttp.web.run_app(app, port=7432)
+6
View File
@@ -0,0 +1,6 @@
import src.libsquare as sq
import src.bricksetapi as lego
oursq = sq.get_itm("71417")
ourlego = lego.get_set(71417)
print(f"{oursq}\n\n{ourlego}\n\n")
+16 -124
View File
@@ -1,27 +1,22 @@
import argparse
import asyncio
import json
from aiohttp import web
from base64 import b64decode
from typing import Dict, Any
import uuid
from time import time
import dotenv
import os
import re
import logging
from logging.handlers import RotatingFileHandler
from pathlib import Path
print("Creating log directory if it doesn't exist...")
os.makedirs("./logs/", exist_ok=True)
Path("./logs/").mkdir(parents=True, exist_ok=True)
import banner
import get_sq_env
from time import time
time = int(time())
import asquare as sq
# from asquareclass import AsyncSquareClient # type: ignore
@@ -35,15 +30,15 @@ from asyncio_throttle.throttler import Throttler
# set up logger
print("Setting up logger...")
logger = logging.getLogger("studspider")
logger = logging.getLogger("edderkop")
logger.setLevel(logging.DEBUG)
handler = RotatingFileHandler("./logs/studspider.log", maxBytes=125000, backupCount=5)
handler = RotatingFileHandler("./logs/edderkop.log", maxBytes=125000, backupCount=5)
logger.addHandler(handler)
lego = AsyncBricksetAPI(debug=True, logger=logger)
# sq = AsyncSquareClient(logger=logger)
# logging.basicConfig(
# filename="./logs/studspider.log", encoding="utf-8", level=logging.DEBUG
# filename="./logs/edderkop.log", encoding="utf-8", level=logging.DEBUG
# )
print("Setting up rate limiter...")
print("Doing RE things...")
@@ -51,19 +46,8 @@ UA = re.compile(r"^kopling\/?", re.IGNORECASE)
async def run_update():
set_list = await sq.acat2id(SQUARE_LEGO_CAT_ID) # type: ignore
# print(f"[@DEBUG]\t {set_list}") # type: ignore
for l_set in set_list:
# print(f"[@DEBUG]{l_set}")
try:
l_setinfo = await lego.throttled_get_set(l_set["number"])
print(f"[@DEBUG] Found {l_setinfo["pw_name"]}")
except Exception as e:
print(f"Error fetching set\t: {l_set}: {str(e)}")
continue
# print(f"[@DEBUG]{l_setinfo}")
# await sq.update_itm(l_set, l_setinfo)
set_list = await sq.acat2id(SQUARE_LEGO_CAT_ID)
print(f"[@DEBUG]\t {set_list}") # type: ignore
"""for l_set in set_list:
print(f"[@DEBUG]{l_set}")
try:
@@ -84,53 +68,14 @@ def valid_uuid(val):
return False
async def hook_handler(request: web.Request) -> web.Response:
"""Handle incoming JSON webhook POSTs on /hook endpoint"""
try:
# Verify request method
if request.method != "POST":
logger.error(
f"{time}\tInvalid request method to /hook from %s", request.remote
)
return web.Response(status=405, text="Method not allowed. Use POST.")
# Get JSON data from request
try:
data = await request.json()
except json.JSONDecodeError:
logger.error("Invalid JSON to /hook from %s", request.remote)
return web.Response(status=400, text="Invalid JSON in request body")
# Log the received webhook
print(f"[@DEBUG] Webhook received at {time} from {request.remote}")
logger.info(
f"{time}\tWebhook received from {request.remote}: {json.dumps(data)}"
)
# Process the webhook data here
# TODO: Add your webhook processing logic
response_data = {
"status": "success",
"message": "Webhook received",
"timestamp": time,
"data_received": data,
}
return web.json_response(response_data)
except Exception as e:
logger.error(f"Webhook handler error: {str(e)}")
print(f"Webhook handler error: {str(e)}")
return web.Response(status=500, text="Internal server error")
async def request_handler(request: web.Request) -> web.Response:
"""Handle incoming POST requests with command processing"""
try:
# Verify request method
if request.method != "POST":
logger.error(f"{time}\tInvalid request method from %s at ", request.remote)
logger.error(
f"{time()}\tInvalid request method from %s at ", request.remote
)
return web.Response(status=405, text="Method not allowed. Use POST.")
# Get JSON data from request
@@ -158,10 +103,10 @@ async def request_handler(request: web.Request) -> web.Response:
try:
decoded_cmd = b64decode(data["CMD"]).decode("utf-8")
print(
f"""At: {time}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
f"""At: {time()}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
)
logging.info(
f"{time}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
f"{time()}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
)
# Process the command
@@ -181,65 +126,12 @@ async def request_handler(request: web.Request) -> web.Response:
print(f"Request handler error: {str(e)}")
return web.Response(status=500, text="Internal server error")
print("Starting web server...")
os.system("cls" if os.name == "nt" else "clear")
os.system('cls' if os.name == 'nt' else 'clear')
app = web.Application()
app.router.add_post("/api", request_handler)
app.router.add_post("/hook", hook_handler)
parser = argparse.ArgumentParser(description="studspider")
parser.add_argument(
"--host",
type=str,
default="localhost",
help="IP address to listen on. default: localhost",
)
parser.add_argument(
"--port", type=int, default=7432, help="Port to listen on. default: 7432"
)
parser.add_argument(
"--hook-port",
type=int,
default=8080,
help="Optional separate port for webhook endpoint. default: 8080",
)
args = parser.parse_args()
app.router.add_post("/", request_handler)
if __name__ == "__main__":
print(banner.generate("studspider v0.1"))
print(banner.generate("Edderkop v0.1"))
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
# If hook-port is specified, run a separate server for webhooks
if args.hook_port:
print(f"Starting main server on {args.host}:{args.port}")
print(f"Starting webhook server on {args.host}:{args.hook_port}")
# Create separate app for webhook
hook_app = web.Application()
hook_app.router.add_post("/hook", hook_handler)
# Run both servers concurrently
async def run_servers():
runner1 = web.AppRunner(app)
await runner1.setup()
site1 = web.TCPSite(runner1, args.host, args.port)
runner2 = web.AppRunner(hook_app)
await runner2.setup()
site2 = web.TCPSite(runner2, args.host, args.hook_port)
await site1.start()
await site2.start()
print(f"Main server running on http://{args.host}:{args.port}")
print(f"Webhook server running on http://{args.host}:{args.hook_port}/hook")
# Keep running
await asyncio.Event().wait()
asyncio.run(run_servers())
else:
print(f"Server running on {args.host}:{args.port}")
print(f"Webhook endpoint: http://{args.host}:{args.port}/hook")
web.run_app(app, host=args.host, port=args.port)
web.run_app(app, host="localhost", port=7432)
-33
View File
@@ -1,33 +0,0 @@
import asyncio
from asyncio_throttle import Throttler as AsyncThrottler
class GlobalThrottler:
_instance = None
_lock = asyncio.Lock()
def __init__(self, rate_limit: int = 500, period: int = 60) -> None:
if not hasattr(self, '_initialized'):
self.rate_limit = rate_limit
self.period = period
self.throttler = AsyncThrottler(rate_limit=self.rate_limit, period=self.period)
self._initialized = True
@classmethod
async def get_instance(cls, rate_limit: int = 500, period: int = 60) -> 'GlobalThrottler':
async with cls._lock:
if not cls._instance:
cls._instance = cls(rate_limit, period)
return cls._instance
async def acquire(self):
await self.throttler.acquire()
def release(self):
self.throttler.release()
async def __aenter__(self):
await self.acquire()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
self.release()
File renamed without changes.
-31
View File
@@ -1,31 +0,0 @@
import sys
from pathlib import Path
from pprint import pprint
sys.path.insert(0, str(Path(__file__).parent.parent))
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
import src.libsquare as sq
import src.bricksetapi as lego
print("=" * 80)
result = sock.connect_ex(('127.0.0.1', 7432))
if result == 0:
print("PORT 7432 IS IN USE")
else:
print("PORT 7432 IS AVAILABLE")
sock.close()
print("=" * 80)
print("SQUARE OBJECT FOR SET 71417:")
print("=" * 80)
oursq = sq.get_itm("71417")
pprint(oursq, width=120, depth=None)
print("\n" + "=" * 80)
print("LEGO SET INFORMATION FOR SET 71417:")
print("=" * 80)
ourlego = lego.get_set(71417)
print("=" * 80)
pprint(ourlego, width=120, depth=None)
Generated
-1136
View File
File diff suppressed because it is too large. Load diff