5 Commits
Author SHA1 Message Date
C b4b0ccdac6 added rotating logs! epicsauce 2025-10-09 13:33:59 -07:00
C 32664c3a10 added optional logging to brickset 2025-10-09 13:33:51 -07:00
C 41ce94c85c Auto stash before rebase of "uv" 2025-10-09 12:55:38 -07:00
ph0sgene 397eddf299 rm 314t 2025-10-09 12:54:32 -07:00
C 8affb90308 refactor: Migrate build system to Hatch and integrate UV 2025-10-09 12:49:37 -07:00
48 changed files with 1180 additions and 7025 deletions

No files matched your search

-29
View File
@@ -1,29 +0,0 @@
# Brickset.com credentials - LEGO database for pricing, images, UPC codes
BRICKSET_USERNAME = "your_brickset_username"
BRICKSET_PASS = "your_brickset_password"
BRICKSET_API_KEY = "your_brickset_api_key"
# Square environment - "sandbox" for testing, "production" for live store
SQUARE_ENV = "sandbox" # or "production"
# Public webhook URL that Square will POST events to
SQUARE_WEBHOOK_URL = "https://yourdomain.com/webhook"
# Sandbox credentials - test environment (fake data, safe for development)
SANDBOX_SQUARE_APP_ID = "your_sandbox_app_id"
SANDBOX_SQUARE_TOKEN = "your_sandbox_access_token"
# Sandbox webhook subscription ID and signature key for verifying webhook requests
SANDBOX_SQUARE_WEBHOOK_SUB_ID = "your_sandbox_webhook_subscription_id"
SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY = "your_sandbox_webhook_signature_key"
# Production credentials - live store (real inventory and sales)
PROD_SQUARE_APP_ID = "your_production_app_id"
PROD_SQUARE_TOKEN = "your_production_access_token"
# Production webhook subscription ID and signature key for verifying webhook requests
PROD_SQUARE_WEBHOOK_SUB_ID = "your_production_webhook_subscription_id"
PROD_SQUARE_WEBHOOK_SIGNATURE_KEY = "your_production_webhook_signature_key"
# Category ID for LEGO items in Square catalog (find in Dashboard → Items → Categories)
SQUARE_LEGO_CAT_ID = "your_lego_category_id"
+26 -43
View File
@@ -1,43 +1,26 @@
logs/
.env
__pycache__/
*.pyc
.vscode/
.idea/
.DS_Store
*.db
*.sqlite3
*.log
bin/build/
.zip
snippet.py
doc/square-src/square-python-sdk-43.0.2.20250820/
doc/square-src/
doc/pw-inventory/pw-inventory.zip
doc/pw-inventory/SQUARE_inventory_forautomation_2025-06-18-0139.csv
doc/pw-inventory/square_inventory_export_2025-06-18-0137.xlsx
square-src.code-workspace
doc/pw-inventory/
studspider.db-journal
.venv
*.db
*.py[cod]
.web
.states
assets/external/
__pycache__/
win/
.github/
bin/
activate/
build/
.pem
.p7b
nginx/
logs/
.env
__pycache__/
*.pyc
.vscode/
.idea/
.DS_Store
*.db
*.sqlite3
*.log
poetry.lock
uv.lock
bin/build/
.zip
snippet.py
doc/square-src/square-python-sdk-43.0.2.20250820/
doc/square-src/
doc/pw-inventory/pw-inventory.zip
doc/pw-inventory/SQUARE_inventory_forautomation_2025-06-18-0139.csv
doc/pw-inventory/square_inventory_export_2025-06-18-0137.xlsx
square-src.code-workspace
-1
View File
@@ -1 +0,0 @@
0.3.0
+11 -43
View File
@@ -1,44 +1,12 @@
# studspider
![Static Badge](https://img.shields.io/badge/python-3.13.7-blue?logo=python) ![Static Badge](https://img.shields.io/badge/code_style-black-black) ![Static Badge](https://img.shields.io/badge/license-MPL-orange)
is a tool used to talk to brickset.com, pull lego set information and automatically enter it into the square inventory!
## Run me
Clone the project
```bash
git clone http://git.traps.moe/ph0sgene/studspider.git
git checkout branch
```
Go to the project directory
```bash
cd studspider
```
Install dependencies
```bash
uv sync && uv lock
```
Start the server
```bash
uv run studspider.py
```
## Tech
- gunicorn
- uvicorn
- aiohttp
- squareup
- apscheduler
# Edderkop
environment: poetry
code style: black
built with
- aiohttp
- squareup
- apscheduler
- aiosqlite
+1 -8
View File
@@ -1,11 +1,4 @@
- glue shit together
- test sq
- test inserting brickowl into sq
- test kopling
square inv needs lego set category changed
key lights
pastel wrist things
some sets
- test kopling
-108
View File
@@ -1,108 +0,0 @@
import multiprocessing
import os
import git
import src.banner as banner
def get_git_commit():
"""Get first 6 characters of current git commit hash"""
try:
repo = git.Repo(search_parent_directories=True)
return repo.head.commit.hexsha[:6]
except Exception:
return "unknown"
# Server socket
bind = "127.0.0.1:7433" # nginx will proxy to this
backlog = 2048
# Worker processes
workers = multiprocessing.cpu_count() * 2 + 1 # Recommended formula
worker_class = "uvicorn.workers.UvicornWorker" # Async worker for FastAPI
worker_connections = 1000
max_requests = 1000 # Restart workers after this many requests
max_requests_jitter = 50 # Add randomness to prevent all workers restarting at once
# Timeouts
timeout = 30 # Workers silent for more than this are killed
keepalive = 2 # Wait this long for requests on Keep-Alive connections
# Logging
accesslog = "./logs/webhook_access.log"
errorlog = "./logs/webhook_error.log"
loglevel = "info"
access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(D)s'
# Process naming
proc_name = "studspider"
# Server mechanics
daemon = False # Run in foreground (systemd will handle daemonization)
pidfile = "./logs/studspider.pid"
umask = 0
user = None # Run as current user (systemd will set user)
group = None
tmp_upload_dir = None
# SSL (not needed with nginx, but here for reference)
# keyfile = None
# certfile = None
# Security
limit_request_line = 4096
limit_request_fields = 100
limit_request_field_size = 8190
# Debugging (disable in production)
reload = os.getenv("GUNICORN_RELOAD", "false").lower() == "true"
reload_engine = "auto"
# Hooks
def on_starting(server):
"""Called just before the master process is initialized."""
print("Studspider WEB starting...")
# Verify .env file exists (workers will load it via get_sq_env.py)
from pathlib import Path
env_path = Path(__file__).parent.parent / ".env"
print(f"[@DEBUG] Looking for .env at: {env_path.absolute()}")
if env_path.exists():
print(f"✓ .env file found (workers will load via get_sq_env.py)")
else:
print(f"⚠ No .env file found at {env_path.absolute()}")
def on_reload(server):
"""Called to recycle workers during a reload via SIGHUP."""
print("🔄 Reloading workers...")
def when_ready(server):
commit_hash = get_git_commit()
banner_text = banner.generate("studspider v0.3.0")
# Add commit hash to the last line of banner
banner_lines = banner_text.rstrip("\n").split("\n")
if banner_lines:
banner_lines[-1] += f" - {commit_hash}"
print("\n".join(banner_lines))
print("Built for PLAY-WELL TEKNOLOGIES by contact@studspider.sh\n\n")
print("PRODUCTION MODE\n" if os.getenv("SQUARE_ENV") == "production" else "SANDBOX MODE\n")
print(f"HOST:\t{bind}")
print(f"WORKERS:\t{workers}")
print(f"WORKER TYPE:\t{worker_class}")
print(f"{'=' * 60}\n")
def worker_int(worker):
"""Called just after a worker exited on SIGINT or SIGQUIT."""
print(f"⚠️ Worker {worker.pid} received INT or QUIT signal")
def worker_abort(worker):
"""Called when a worker received the SIGABRT signal."""
print(f"❌ Worker {worker.pid} aborted")
-136
View File
@@ -1,136 +0,0 @@
# Understanding PEM vs P7B Certificates
## Quick Summary
**You CANNOT use P7B files to run your HTTPS server!**
- **PEM files** = Run your server with HTTPS ✓
- **P7B files** = Upload to Square Dashboard ✓
## The Complete Picture
```
┌─────────────────────────────────────────────────────────────┐
│ Your HTTPS Server │
│ │
│ Uses: cert.pem + key.pem (PEM format) │
│ Command: python src/test_server.py │
│ --ssl-cert cert.pem │
│ --ssl-key key.pem │
│ --hook-port 8443 │
└─────────────────────────────────────────────────────────────┘
│
│ HTTPS traffic
▼
┌─────────────────────────────────────────────────────────────┐
│ Square's Servers │
│ │
│ They verify your server using: cert.p7b │
│ (which YOU upload to Square Dashboard) │
└─────────────────────────────────────────────────────────────┘
```
## Why Two Formats?
### PEM Format (.pem files)
- **Contains**: Certificate AND private key (separate files)
- **Used by**: Your HTTPS server to encrypt traffic
- **Files needed**:
- `cert.pem` - Your public certificate
- `key.pem` - Your private key (NEVER share this!)
### P7B Format (.p7b files)
- **Contains**: Certificate ONLY (no private key)
- **Used by**: Square to verify webhook signatures
- **File**: `cert.p7b` - Uploaded to Square Dashboard
## Step-by-Step Workflow
### 1. Generate Certificates (PEM format)
```powershell
python generate_cert.py
```
**Creates:**
- `cert.pem` - Your certificate (public)
- `key.pem` - Your private key (secret!)
### 2. Start Your HTTPS Server (uses PEM)
```powershell
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only
```
**Server runs at:**
- `https://localhost:8443/hook`
### 3. Convert Certificate to P7B for Square
```powershell
python convert_to_p7b.py
```
**Creates:**
- `cert.p7b` - Certificate in P7B format (for Square)
### 4. Upload P7B to Square
1. Go to: https://developer.squareup.com/apps
2. Your App → **Webhooks** → **Manage Notification Settings**
3. Upload `cert.p7b`
### 5. Configure Square Webhook URL
- Enter: `https://yourdomain.com:8443/hook`
- Square will verify using the P7B you uploaded
## Common Mistakes
### ❌ Trying to use P7B with your server
```powershell
# THIS WON'T WORK!
python src/test_server.py --ssl-cert cert.p7b --ssl-key key.p7b
```
**Error:** P7B doesn't contain the private key needed for HTTPS!
### ✓ Correct usage
```powershell
# Server uses PEM files
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443
# P7B file is uploaded to Square Dashboard (not used by your server)
```
## If You Only Have a P7B File
If someone gave you a P7B file, you can extract the certificate:
```powershell
python convert_from_p7b.py your_cert.p7b --output cert.pem
```
**BUT:** You still need the private key (`key.pem`) to run HTTPS!
If you don't have the private key, you must:
1. Generate a new certificate pair: `python generate_cert.py`
2. Or get the private key from whoever created the P7B
## File Checklist
To run your HTTPS server with Square webhooks, you need:
- [ ] `cert.pem` - Certificate (PEM format) - For your server
- [ ] `key.pem` - Private key (PEM format) - For your server
- [ ] `cert.p7b` - Certificate (P7B format) - Upload to Square
Generate all three with:
```powershell
python generate_cert.py # Creates cert.pem and key.pem
python convert_to_p7b.py # Creates cert.p7b from cert.pem
```
## Security Notes
- **Never share `key.pem`** - This is your private key!
- **`cert.pem` is public** - Safe to share
- **`cert.p7b` is public** - You upload this to Square
- Self-signed certificates are fine for testing but use Let's Encrypt for production
## Need Help?
See the detailed guides:
- `HTTPS_SETUP.md` - Complete HTTPS setup instructions
- `SQUARE_WEBHOOK_SETUP.md` - Square webhook configuration
-166
View File
@@ -1,166 +0,0 @@
# HTTPS Webhook Setup Guide
## Quick Start (Self-Signed Certificate for Testing)
### 1. Generate Certificate
```powershell
python generate_cert.py
```
This creates `cert.pem` and `key.pem` files.
### 1.5. Convert to P7B for Square (Required for Square Webhooks)
```powershell
python convert_to_p7b.py
```
This creates `cert.p7b` which Square needs for webhook signature verification.
Upload this P7B file to Square Developer Dashboard → Webhooks → Manage Notification Settings.
### 2. Run Server with HTTPS
#### Option A: HTTPS for webhook only (recommended for development)
```powershell
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only
```
- Main server: `http://localhost:7432/` (HTTP)
- Webhook: `https://localhost:8443/hook` (HTTPS)
#### Option B: HTTPS for everything
```powershell
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443
```
- Main server: `https://localhost:7432/` (HTTPS)
- Webhook: `https://localhost:8443/hook` (HTTPS)
#### Option C: Single port with HTTPS
```powershell
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem
```
- Everything on: `https://localhost:7432/`
### 3. Test the Webhook
Update `src/test_webhook.py` to use HTTPS:
```python
url = "https://localhost:8443/hook"
```
Then run:
```powershell
python src/test_webhook.py
```
## Production Setup (with Real Certificate)
### Option 1: Let's Encrypt (Free, Recommended)
On Linux/WSL:
```bash
# Install certbot
sudo apt-get install certbot
# Generate certificate (replace yourdomain.com)
sudo certbot certonly --standalone -d yourdomain.com
# Certificates will be at:
# /etc/letsencrypt/live/yourdomain.com/fullchain.pem
# /etc/letsencrypt/live/yourdomain.com/privkey.pem
# Run server
python src/test_server.py \
--ssl-cert /etc/letsencrypt/live/yourdomain.com/fullchain.pem \
--ssl-key /etc/letsencrypt/live/yourdomain.com/privkey.pem \
--host 0.0.0.0 \
--hook-port 443 \
--hook-ssl-only
```
### Option 2: Reverse Proxy (nginx/Caddy)
Run studspider on HTTP internally, use nginx/Caddy for HTTPS:
```nginx
# nginx config
server {
listen 443 ssl;
server_name yourdomain.com;
ssl_certificate /path/to/cert.pem;
ssl_certificate_key /path/to/key.pem;
location /hook {
proxy_pass http://localhost:7432/hook;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
}
}
```
Then run studspider normally:
```powershell
python src/test_server.py
```
## Troubleshooting
### "OpenSSL not found" Error
- **Windows**: Install from https://slproweb.com/products/Win32OpenSSL.html
- **Alternative**: Use Git Bash or WSL which include OpenSSL
### Self-Signed Certificate Warnings
- Normal for development
- Use `--insecure` flag with curl/tools for testing
- For production, get a proper certificate from Let's Encrypt
### Port 443 Already in Use
- Use a different port like 8443
- Or stop the service using port 443
### Certificate File Not Found
- Use absolute paths: `--ssl-cert C:\path\to\cert.pem`
- Or run from the directory containing the certificates
## Square Webhook Configuration
### Certificate Requirements
Square requires a **P7B (PKCS#7)** certificate file for webhook signature verification.
### Setup Steps:
1. **Generate your certificate** (if you haven't already):
```powershell
python generate_cert.py
```
2. **Convert PEM to P7B format**:
```powershell
# Basic conversion
python convert_to_p7b.py
# With intermediate certificates (production)
python convert_to_p7b.py --cert fullchain.pem --chain intermediate.pem --output square.p7b
```
3. **Upload to Square Dashboard**:
- Go to https://developer.squareup.com/
- Navigate to your application
- Click on "Webhooks"
- Click "Manage Notification Settings"
- Upload the `cert.p7b` file
- Square will use this to verify webhook signatures
4. **Configure webhook URL in Square**:
- Use your public HTTPS URL: `https://yourdomain.com:8443/hook`
- Square requires valid SSL (not self-signed for production)
### Testing with Square Webhooks:
When setting up webhooks in Square Dashboard:
1. Use your HTTPS URL: `https://yourdomain.com:8443/hook`
2. Square requires valid SSL (not self-signed)
3. Consider using ngrok for local testing:
```powershell
ngrok http 8443
```
Then use the ngrok HTTPS URL in Square Dashboard
-356
View File
@@ -1,356 +0,0 @@
# Nginx Reverse Proxy Configuration for studspider
## Overview
When using nginx as a reverse proxy:
- **Nginx handles HTTPS** (SSL/TLS termination)
- **studspider runs on HTTP** internally (no SSL needed)
- **nginx forwards requests** to studspider
## Architecture
```
Internet (HTTPS) → nginx (443) → studspider (7432/HTTP)
↓
Handles SSL certificates
Handles HTTPS traffic
Forwards to HTTP backend
```
## Benefits
✓ **No SSL certificates needed in studspider**
✓ **nginx handles SSL efficiently**
✓ **Easy certificate renewal** (Let's Encrypt)
✓ **Better performance** (nginx is optimized for this)
✓ **Additional security** (rate limiting, IP filtering, etc.)
---
## Configuration Examples
### Basic Configuration (HTTP → HTTPS redirect + webhook proxy)
Create `/etc/nginx/sites-available/studspider`:
```nginx
# Redirect HTTP to HTTPS
server {
listen 80;
server_name studspider.sh www.studspider.sh;
location / {
return 301 https://$host$request_uri;
}
}
# HTTPS server
server {
listen 443 ssl http2;
server_name studspider.sh www.studspider.sh;
# SSL certificates (Let's Encrypt)
ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
# SSL configuration (modern, secure)
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
# Logs
access_log /var/log/nginx/studspider_access.log;
error_log /var/log/nginx/studspider_error.log;
# Webhook endpoint (Square webhooks)
location /hook {
proxy_pass http://localhost:7432/hook;
# Forward headers
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Timeouts
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
# Disable buffering for real-time responses
proxy_buffering off;
}
# Main API endpoint (optional)
location / {
proxy_pass http://localhost:7432/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
```
### Separate Port Configuration (webhook on different port)
```nginx
server {
listen 443 ssl http2;
server_name studspider.sh;
ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
# Main API
location / {
proxy_pass http://localhost:7432/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
}
}
# Separate server for webhooks on different port
server {
listen 8443 ssl http2;
server_name studspider.sh;
ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
# Webhook only
location /hook {
proxy_pass http://localhost:8080/hook; # studspider webhook port
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
}
}
```
### With Rate Limiting (recommended for webhooks)
```nginx
# Define rate limit zone (outside server block)
limit_req_zone $binary_remote_addr zone=webhook_limit:10m rate=10r/s;
server {
listen 443 ssl http2;
server_name studspider.sh;
ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
# Webhook with rate limiting
location /hook {
# Rate limit: 10 requests per second per IP
limit_req zone=webhook_limit burst=20 nodelay;
# Only allow POST requests
limit_except POST {
deny all;
}
proxy_pass http://localhost:7432/hook;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
}
}
```
---
## Setup Steps
### 1. Install nginx (if not already installed)
```bash
# Ubuntu/Debian
sudo apt update
sudo apt install nginx
# CentOS/RHEL
sudo yum install nginx
```
### 2. Get SSL Certificate (Let's Encrypt - Free!)
```bash
# Install certbot
sudo apt install certbot python3-certbot-nginx
# Get certificate for your domain
sudo certbot --nginx -d studspider.sh -d www.studspider.sh
# Certbot will automatically:
# 1. Generate certificate
# 2. Configure nginx
# 3. Set up auto-renewal
```
### 3. Create nginx Configuration
```bash
# Create config file
sudo nano /etc/nginx/sites-available/studspider
# Paste one of the configurations above
# Create symbolic link to enable site
sudo ln -s /etc/nginx/sites-available/studspider /etc/nginx/sites-enabled/
# Test configuration
sudo nginx -t
# Reload nginx
sudo systemctl reload nginx
```
### 4. Start studspider (HTTP only, no SSL needed!)
```bash
# Simple - single port
python src/test_server.py --port 7432
# Or with separate webhook port
python src/test_server.py --port 7432 --hook-port 8080
```
### 5. Configure Square Webhook URL
In Square Developer Dashboard:
- **Webhook URL**: `https://studspider.sh/hook`
- **Certificate**: Upload the P7B file (see below)
### 6. Generate P7B for Square
```bash
# Convert your Let's Encrypt certificate to P7B for Square
python convert_to_p7b.py --cert /etc/letsencrypt/live/studspider.sh/fullchain.pem --output square.p7b
# Upload square.p7b to Square Dashboard
```
---
## Systemd Service (Run studspider at boot)
Create `/etc/systemd/system/studspider.service`:
```ini
[Unit]
Description=studspider - LEGO Inventory Sync Service
After=network.target
[Service]
Type=simple
User=www-data
WorkingDirectory=/opt/studspider
Environment="PATH=/opt/studspider/.venv/bin"
ExecStart=/opt/studspider/.venv/bin/python src/test_server.py --port 7432
Restart=always
RestartSec=10
# Logging
StandardOutput=append:/var/log/studspider/stdout.log
StandardError=append:/var/log/studspider/stderr.log
[Install]
WantedBy=multi-user.target
```
Enable and start:
```bash
# Create log directory
sudo mkdir -p /var/log/studspider
sudo chown www-data:www-data /var/log/studspider
# Enable service
sudo systemctl enable studspider
# Start service
sudo systemctl start studspider
# Check status
sudo systemctl status studspider
# View logs
sudo journalctl -u studspider -f
```
---
## Testing
### Test nginx configuration
```bash
sudo nginx -t
```
### Test studspider locally
```bash
curl http://localhost:7432/hook -X POST -H "Content-Type: application/json" -d '{"test":"data"}'
```
### Test through nginx
```bash
curl https://studspider.sh/hook -X POST -H "Content-Type: application/json" -d '{"test":"data"}'
```
### Monitor logs
```bash
# nginx logs
sudo tail -f /var/log/nginx/studspider_access.log
sudo tail -f /var/log/nginx/studspider_error.log
# studspider logs
tail -f ./logs/studspider.log
```
---
## Troubleshooting
### 502 Bad Gateway
- Check if studspider is running: `ps aux | grep test_server.py`
- Check the port matches nginx config
- Check firewall isn't blocking internal connections
### Certificate Errors
- Ensure Let's Encrypt renewal is working: `sudo certbot renew --dry-run`
- Check certificate files exist and have correct permissions
### Webhook Not Receiving
- Verify nginx is forwarding: Check nginx access logs
- Verify studspider is receiving: Check studspider logs
- Test locally first, then through nginx
### Connection Refused
- Check studspider is listening: `sudo netstat -tlnp | grep 7432`
- Check nginx proxy_pass points to correct host:port
---
## Summary
### With nginx reverse proxy:
✓ **studspider runs simple HTTP** (no SSL certificates needed)
```bash
python src/test_server.py --port 7432
```
✓ **nginx handles HTTPS** (manages certificates automatically)
✓ **Square gets HTTPS webhook** (`https://studspider.sh/hook`)
✓ **You upload P7B to Square** (converted from nginx's certificate)
### This is the recommended production setup!
-191
View File
@@ -1,191 +0,0 @@
# Square Webhook Setup - Quick Reference
## Complete Setup Process
### Step 1: Generate SSL Certificate
```powershell
# Generate PEM certificate
python generate_cert.py
# Creates: cert.pem, key.pem
```
### Step 2: Convert to P7B for Square
```powershell
# Convert PEM to P7B (PKCS#7) format
python convert_to_p7b.py
# Creates: cert.p7b
```
### Step 3: Start HTTPS Server
```powershell
# Start webhook server with HTTPS
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only
# Server URLs:
# Main: http://localhost:7432/
# Webhook: https://localhost:8443/hook (or https://yourdomain.com:8443/hook)
```
### Step 4: Configure Square Webhook
1. **Upload P7B Certificate to Square:**
- Go to: https://developer.squareup.com/apps
- Select your application
- Navigate to: **Webhooks** → **Manage Notification Settings**
- Upload `cert.p7b` file
- This allows Square to verify webhook signatures
2. **Add Webhook Subscription:**
- Still in Webhooks section
- Click **"Add Subscription"**
- Enter URL: `https://yourdomain.com:8443/hook`
- Select events to subscribe to (e.g., `inventory.count.updated`)
- Click **Save**
### Step 5: Test the Webhook
```powershell
# Test from command line
python src/test_webhook.py
# Or manually with curl
curl -X POST https://localhost:8443/hook \
-H "Content-Type: application/json" \
-d '{"type":"inventory.count.updated","merchant_id":"test"}'
```
## Production Deployment
### Using Let's Encrypt (Recommended)
```bash
# 1. Get Let's Encrypt certificate (Linux/WSL)
sudo certbot certonly --standalone -d yourdomain.com
# 2. Convert to P7B for Square
python convert_to_p7b.py \
--cert /etc/letsencrypt/live/yourdomain.com/fullchain.pem \
--output square.p7b
# 3. Upload square.p7b to Square Dashboard
# 4. Run server
python src/test_server.py \
--ssl-cert /etc/letsencrypt/live/yourdomain.com/fullchain.pem \
--ssl-key /etc/letsencrypt/live/yourdomain.com/privkey.pem \
--host 0.0.0.0 \
--hook-port 443 \
--hook-ssl-only
```
### Using ngrok (Easy Testing)
```powershell
# Terminal 1: Start your server (HTTP is fine)
python src/test_server.py --hook-port 8080
# Terminal 2: Expose with ngrok (provides HTTPS automatically)
ngrok http 8080
# Use the ngrok HTTPS URL in Square:
# Example: https://abc123.ngrok.io/hook
# For P7B certificate with ngrok:
# ngrok provides its own certificate, so you don't need to upload one to Square
```
## Webhook Event Types
Common Square webhook events you can subscribe to:
- `inventory.count.updated` - Inventory quantity changes
- `catalog.version.updated` - Catalog item updates
- `payment.created` - New payment received
- `payment.updated` - Payment status changes
- `order.created` - New order created
- `order.updated` - Order status changes
## Troubleshooting
### Certificate Upload Fails
- Ensure file is in P7B format (not PEM)
- File must be valid PKCS#7 structure
- Check file isn't corrupted: `openssl pkcs7 -print_certs -in cert.p7b`
### Webhook Not Receiving Data
1. Check server is running and accessible from internet
2. Verify URL in Square Dashboard is correct
3. Check firewall allows incoming traffic on webhook port
4. Review server logs: `./logs/studspider.log`
### SSL Certificate Errors
- Self-signed certificates won't work with Square in production
- Use Let's Encrypt or commercial CA for production
- Use ngrok for quick testing (provides valid HTTPS)
### Testing Locally
```powershell
# Square can't reach localhost - use one of these:
# 1. ngrok (easiest)
ngrok http 8443
# 2. Cloudflare Tunnel
cloudflared tunnel --url http://localhost:8443
# 3. Deploy to cloud (AWS, Azure, DigitalOcean, etc.)
```
## Verifying Webhook Signatures
Square signs webhook requests. Your server should verify signatures:
```python
# TODO: Add to hook_handler in test_server.py
import hmac
import hashlib
def verify_square_signature(payload: bytes, signature: str, webhook_signature_key: str) -> bool:
"""Verify Square webhook signature"""
computed_signature = hmac.new(
webhook_signature_key.encode('utf-8'),
payload,
hashlib.sha256
).digest()
expected_signature = base64.b64decode(signature)
return hmac.compare_digest(computed_signature, expected_signature)
```
Get your webhook signature key from Square Dashboard → Webhooks → Signature Key.
## Environment Variables
Add to your `.env` file:
```env
# Square Webhook Configuration
SQUARE_WEBHOOK_SIGNATURE_KEY=your_signature_key_from_square_dashboard
# SSL Certificate Paths (production)
SSL_CERT_PATH=/path/to/fullchain.pem
SSL_KEY_PATH=/path/to/privkey.pem
```
## Commands Quick Reference
```powershell
# Generate certificate
python generate_cert.py
# Convert to P7B
python convert_to_p7b.py
# Start HTTPS server
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only
# Test webhook
python src/test_webhook.py
# Start with ngrok
ngrok http 8443
```
+11
View File
@@ -0,0 +1,11 @@
{
"folders": [
{
"path": "."
},
{
"path": "../kopling"
}
],
"settings": {}
}
+14
View File
@@ -0,0 +1,14 @@
import src.test_server
class edderkop:
def __init__(self):
self.name = "Edderkop"
self.version = "0.0.1"
self.description = """
Edderkop is an application designed to interact with the Brickset API and directly with Square.
It allows users to search for items, manage collections, and perform various operations related to bricksets.
"""
def run(self):
print(f"{self.name} v{self.version} is running.")
Binary file not shown.

Before

Width:  |  Height:  |  Size: 68 KiB

+32 -39
View File
@@ -1,39 +1,32 @@
[project]
name = "studpider"
version = "0.3.0"
description = ""
authors = [{ name = "CF", email = "ph0sgene@pm.me" }]
requires-python = ">=3.13,<3.14"
readme = "README.md"
license = { text = "MIT" }
dependencies = [
"aiohttp (>=3.12.13,<4.0.0)",
"dotenv (>=0.9.9,<0.10.0)",
"pyinstaller (>=6.14.1,<7.0.0)",
"squareup(>=43.0.0.20250618,<44.0.0.0)",
"asyncio-throttle (>=1.0.2,<2.0.0)",
"requests (>=2.32.4,<3.0.0)",
"pytest (>=8.4.2,<9.0.0)",
"apscheduler (>=3.11.0,<4.0.0)",
"aiosqlite (>=0.21.0,<0.22.0)",
"bs4>=0.0.2",
"reflex>=0.8.14.post1",
"fastapi>=0.128.0",
"uvicorn>=0.40.0",
"gunicorn>=23.0.0",
"async-lru>=2.0.4",
"gitpython>=3.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.uv]
package = false
[tool.bricksetapi]
version = "0.1.0"
[tool.abricksetapi]
version = "0.1.0"
[project]
name = "edderkop"
version = "0.1.0"
description = ""
authors = [{ name = "CF", email = "ph0sgene@pm.me" }]
requires-python = ">=3.13,<3.14"
readme = "README.md"
license = { text = "MIT" }
dependencies = [
"aiohttp (>=3.12.13,<4.0.0)",
"dotenv (>=0.9.9,<0.10.0)",
"pyinstaller (>=6.14.1,<7.0.0)",
"squareup(>=43.0.0.20250618,<44.0.0.0)",
"asyncio-throttle (>=1.0.2,<2.0.0)",
"requests (>=2.32.4,<3.0.0)",
"pytest (>=8.4.2,<9.0.0)",
"apscheduler (>=3.11.0,<4.0.0)",
"aiosqlite (>=0.21.0,<0.22.0)",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.uv]
package = false
[tool.bricksetapi]
version = "0.1.0"
[tool.abricksetapi]
version = "0.1.0"
+10 -10
View File
@@ -1,19 +1,19 @@
from setuptools import setup
setup(
name="studspider",
version="0.3.0",
description="Bricket + Square integration application",
author="CF",
author_email="chris-forst@pm.me",
packages=["src"],
package_dir={"": "src"},
name='Edderkop',
version='0.1',
description='Bricket + Square integration application',
author='CF',
author_email='chris-forst@pm.me',
packages=['src'],
package_dir={'': 'src'},
install_requires=[
# List your project dependencies here
],
entry_points={
"console_scripts": [
"main=main:main", # Adjust this line based on your main function
'console_scripts': [
'main=main:main', # Adjust this line based on your main function
],
},
)
)
View File
Whitespace-only changes.
+120 -120
View File
@@ -1,120 +1,120 @@
import aiohttp
import json
import dotenv
import os
import importlib.metadata
dotenv.load_dotenv()
api_key = os.getenv("BRICKOWL")
abrickowl_version = importlib.metadata.version("abrickowlapi")
class ABrickowlAPI:
def __init__(self, value):
self.user_agent = f"abrickowlapi/{abrickowl_version}"
self.headers = {"content-type": "application/x-www-form-urlencoded"}
self.base_url = "https://api.brickowl.com/v1/"
self.session = aiohttp.ClientSession()
return self
def set_valid(self, set_id: int):
"""
Verify if a LEGO set ID is valid.
"""
if len(set_id) == 5:
return True
else:
return False
@classmethod
async def catalog_list(self):
"""
Fetch the list of LEGO sets from the Brickowl API.
"""
async with self.session.get(
f"{self.base_url}catalog/list/view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Error fetching list, {response.status}")
@classmethod
async def catalog_lookup(self, set_id: int):
"""
Fetch details from the brickowl API for a specific LEGO set.
set_id: The LEGO set ID to look up.
returns JSON response containing set details.
"""
async with self.session.get(
f"{self.base_url}catalog/lookup/{set_id}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Error fetching set {set_id}, {response.status}")
@classmethod
async def catalog_availability(self, set_id: int):
"""
Fetch availability information for a specific LEGO set.
"""
async with self.session.get(
f"{self.base_url}catalog/availability/{set_id}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(
f"Error fetching fetching availability for {set_id}, {response.status}"
)
@classmethod
async def catalog_id_lookup(self, set_id: int):
async with self.session.get(
f"{self.base_url}catalog/id_lookup/{set_id}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"ID lookup failed for set {set_id}, {response.status}")
@classmethod
async def catalog_bulk(self):
async with self.session.get(
f"{self.base_url}catalog/bulk/view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Error fetching bulk, {response.status}")
@classmethod
async def catalog_search(self, query: str):
async with self.session.get(
f"{self.base_url}catalog/search/{query}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Search term {set_id} failed, {response.status}")
# Usage:
# obj = await ABrickowlAPI.catalog_lookup(value)
import aiohttp
import json
import dotenv
import os
import importlib.metadata
dotenv.load_dotenv()
api_key = os.getenv("BRICKOWL")
abrickowl_version = importlib.metadata.version("abrickowlapi")
class ABrickowlAPI:
def __init__(self, value):
self.user_agent = f"abrickowlapi/{abrickowl_version}"
self.headers = {"content-type": "application/x-www-form-urlencoded"}
self.base_url = "https://api.brickowl.com/v1/"
self.session = aiohttp.ClientSession()
return self
def set_valid(self, set_id: int):
"""
Verify if a LEGO set ID is valid.
"""
if len(set_id) == 5:
return True
else:
return False
@classmethod
async def catalog_list(self):
"""
Fetch the list of LEGO sets from the Brickowl API.
"""
async with self.session.get(
f"{self.base_url}catalog/list/view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Error fetching list, {response.status}")
@classmethod
async def catalog_lookup(self, set_id: int):
"""
Fetch details from the brickowl API for a specific LEGO set.
set_id: The LEGO set ID to look up.
returns JSON response containing set details.
"""
async with self.session.get(
f"{self.base_url}catalog/lookup/{set_id}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Error fetching set {set_id}, {response.status}")
@classmethod
async def catalog_availability(self, set_id: int):
"""
Fetch availability information for a specific LEGO set.
"""
async with self.session.get(
f"{self.base_url}catalog/availability/{set_id}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(
f"Error fetching fetching availability for {set_id}, {response.status}"
)
@classmethod
async def catalog_id_lookup(self, set_id: int):
async with self.session.get(
f"{self.base_url}catalog/id_lookup/{set_id}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"ID lookup failed for set {set_id}, {response.status}")
@classmethod
async def catalog_bulk(self):
async with self.session.get(
f"{self.base_url}catalog/bulk/view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Error fetching bulk, {response.status}")
@classmethod
async def catalog_search(self, query: str):
async with self.session.get(
f"{self.base_url}catalog/search/{query}view?key={api_key}",
headers=self.headers,
user_agent=self.user_agent,
) as response:
if response.status == 200:
return await response.json()
else:
raise Exception(f"Search term {set_id} failed, {response.status}")
# Usage:
# obj = await ABrickowlAPI.catalog_lookup(value)
+159 -280
View File
@@ -1,280 +1,159 @@
"""
abricksetapi.py - Asynchronous Brickset API Client
This module provides asynchronous interactions with the Brickset API.
"""
import asyncio
import os
import aiohttp
import json
import math
import dotenv
from typing import Dict, Any, Optional
from datetime import datetime, timedelta
from async_lru import alru_cache
from . import throttler
from . import get_sq_env
Throttler = throttler.GlobalThrottler
dotenv.load_dotenv()
class AsyncBricksetAPI:
"""Asynchronous client for the Brickset API"""
def __init__(self, debug: bool = None, logger=None):
self.username = os.getenv("BRICKSET_USERNAME")
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
# Use DEBUG_MODE from env if debug not explicitly set
self.debug = debug if debug is not None else get_sq_env.DEBUG_MODE
self.headers = {"UserAgent": "studspider/0.1"}
self.throttler = Throttler(rate_limit=500, period=60)
self.user_hash = None
self.logger = logger
# API usage tracking
self.daily_requests_used: Optional[int] = None
self.daily_requests_limit = 1000
self.usage_last_checked: Optional[datetime] = None
self.usage_check_interval = timedelta(hours=1) # Check usage every hour
async def get_key_usage_stats(self) -> Dict[str, Any]:
"""
Fetch API key usage statistics from Brickset.
Returns:
dict: Usage stats including dateStamp and apiCalls
"""
url = "https://brickset.com/api/v3.asmx/getKeyUsageStats"
params = {"apiKey": self.api_key}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers
) as response:
if response.status == 200:
data = await response.read()
result = json.loads(data.decode("utf-8"))
if self.debug:
print(f"[@DEBUG] API usage stats: {result}")
if self.logger:
self.logger.debug(f"API usage stats: {result}")
return result
else:
error_msg = f"Failed to get usage stats: HTTP {response.status}"
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
async def check_and_update_usage(self) -> bool:
"""
Check current API usage and update internal counter.
Automatically called before each request if needed.
Returns:
bool: True if under limit, False if at/over limit
"""
now = datetime.now()
# Check usage if we haven't checked yet or if it's been over an hour
should_check = (
self.daily_requests_used is None
or self.usage_last_checked is None
or (now - self.usage_last_checked) > self.usage_check_interval
)
if should_check:
try:
stats = await self.get_key_usage_stats()
# Brickset returns {"status": "success", "apiKeyUsage": [...]}
usage_array = stats.get("apiKeyUsage", [])
if usage_array and len(usage_array) > 0:
today_stats = usage_array[0] # Most recent day
self.daily_requests_used = today_stats.get("count", 0)
self.usage_last_checked = now
if self.logger:
self.logger.info(
f"API usage: {self.daily_requests_used}/{self.daily_requests_limit}"
)
if self.debug:
print(
f"[@DEBUG] API usage: {self.daily_requests_used}/{self.daily_requests_limit}"
)
except Exception as e:
if self.logger:
self.logger.warning(f"Failed to check API usage: {e}")
# Don't block requests if usage check fails
return True
# Check if we're over the limit
if self.daily_requests_used is not None:
if self.daily_requests_used >= self.daily_requests_limit:
error_msg = f"Daily API limit reached: {self.daily_requests_used}/{self.daily_requests_limit}"
if self.logger:
self.logger.error(error_msg)
if self.debug:
print(f"[@ERROR] {error_msg}")
return False
# Warn when approaching limit
if self.daily_requests_used >= self.daily_requests_limit * 0.9:
warning_msg = f"Approaching daily API limit: {self.daily_requests_used}/{self.daily_requests_limit}"
if self.logger:
self.logger.warning(warning_msg)
if self.debug:
print(f"[@WARNING] {warning_msg}")
return True
async def login(self) -> Dict[str, Any]:
"""Asynchronously authenticate with Brickset API"""
url = "https://brickset.com/api/v3.asmx/login"
params = {
"username": self.username,
"password": self.password,
"apiKey": self.api_key,
}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers # type: ignore
) as response:
if self.debug:
print(f"[@DEBUG] Login response status: {response.status}")
if self.logger:
self.logger.debug(f"Login response status: {response.status}")
data = await response.read()
result = json.loads(data.decode("utf-8"))
self.user_hash = result.get("hash")
if self.logger:
self.logger.info("Successfully authenticated with Brickset API")
return result
@alru_cache(maxsize=1000, ttl=604800) # Cache 1000 sets, 7 days TTL
async def get_set(self, set_number: int) -> Dict[str, Any]:
"""
Fetch LEGO set information asynchronously (cached for 7 days)
Args:
set_number (str): LEGO set number
Returns:
Dict containing set information
Raises:
Exception: If daily API limit exceeded or API error
"""
# Check API usage before making request
if not await self.check_and_update_usage():
raise Exception(
f"Brickset API daily limit reached ({self.daily_requests_limit} requests/day). "
"Please try again tomorrow."
)
if not self.user_hash:
await self.login()
url = "https://brickset.com/api/v3.asmx/getSets"
params = {
"apiKey": self.api_key,
"userHash": self.user_hash,
"params": json.dumps({"setNumber": f"{set_number}-1"}),
}
print(f"[@DEBUG] {params}...")
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers
) as response:
if self.debug:
print(f"[@DEBUG] Get set response status: {response.status}")
if self.logger:
self.logger.debug(f"Get set response status: {response.status}")
if response.status == 200:
data = await response.read()
result = json.loads(data.decode("utf-8"))
if not result.get("sets"):
error_msg = f"No data found for set {set_number}"
if self.logger:
self.logger.error(error_msg)
raise ValueError(error_msg)
if self.logger:
self.logger.info(
f"Successfully retrieved data for set {set_number}"
)
set_data = result["sets"][0]
# Try to get UPC barcode, fall back to EAN if not available
barcode_data = set_data.get("barcode", {})
upc_code = barcode_data.get("UPC") or barcode_data.get("EAN")
# Get retail price before rounding
retail_price = set_data["LEGOCom"]["US"]["retailPrice"]
our_set = {
"set_num": set_data["number"],
"set_name": set_data["name"],
"set_theme": set_data["theme"],
"set_image": set_data["image"]["thumbnailURL"],
"set_barcode": upc_code,
"set_retailprice": round(retail_price),
}
# Calculate adjusted price (10% markup), then round up
adjusted_price = retail_price * 1.10
our_set["set_adjustredprice"] = math.ceil(adjusted_price)
# Set PW inventory name with theme
our_set["pw_name"] = (
f"{our_set['set_num']} - {our_set['set_theme'].upper()} - {our_set['set_name']}"
)
# Flag for manual price review if needed
if our_set["set_retailprice"] >= 110:
our_set["requires_manual_price"] = True
return our_set
elif response.status == 500:
error_msg = (
"Internal Server Error: Please check API key and parameters."
)
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
else:
error_msg = f"Unexpected response: {response.status}"
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
async def throttled_get_set(self, set_number: str) -> Dict[str, Any]:
"""Get set information with rate limiting"""
async with self.throttler:
return await self.get_set(set_number)
async def main():
"""Example usage"""
client = AsyncBricksetAPI(debug=True)
try:
set_info = await client.throttled_get_set("10349")
print(json.dumps(set_info, indent=2))
except Exception as e:
print(f"Error: {e}")
if __name__ == "__main__":
asyncio.run(main())
"""
abricksetapi.py - Asynchronous Brickset API Client
This module provides asynchronous interactions with the Brickset API.
"""
import asyncio
import os
import aiohttp
import json
import dotenv
from typing import Dict, Any
from asyncio_throttle.throttler import Throttler
dotenv.load_dotenv()
class AsyncBricksetAPI:
"""Asynchronous client for the Brickset API"""
def __init__(self, debug: bool = False, logger=None):
self.username = os.getenv("BRICKSET_USERNAME")
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
self.debug = debug
self.headers = {"UserAgent": "Edderkop/0.1"}
self.throttler = Throttler(rate_limit=500, period=60)
self.user_hash = None
self.logger = logger
async def login(self) -> Dict[str, Any]:
"""Asynchronously authenticate with Brickset API"""
url = "https://brickset.com/api/v3.asmx/login"
params = {
"username": self.username,
"password": self.password,
"apiKey": self.api_key,
}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers # type: ignore
) as response:
if self.debug:
print(f"[@DEBUG] Login response status: {response.status}")
if self.logger:
self.logger.debug(f"Login response status: {response.status}")
data = await response.read()
result = json.loads(data.decode("utf-8"))
self.user_hash = result.get("hash")
if self.logger:
self.logger.info("Successfully authenticated with Brickset API")
return result
async def get_set(self, set_number: str) -> Dict[str, Any]:
"""
Fetch LEGO set information asynchronously
Args:
set_number (str): LEGO set number
Returns:
Dict containing set information
"""
if not self.user_hash:
await self.login()
url = "https://brickset.com/api/v3.asmx/getSets"
params = {
"apiKey": self.api_key,
"userHash": self.user_hash,
"params": json.dumps({"setNumber": f"{set_number}-1"}),
}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers
) as response:
if self.debug:
print(f"[@DEBUG] Get set response status: {response.status}")
if self.logger:
self.logger.debug(f"Get set response status: {response.status}")
if response.status == 200:
data = await response.read()
result = json.loads(data.decode("utf-8"))
if not result.get("sets"):
error_msg = f"No data found for set {set_number}"
if self.logger:
self.logger.error(error_msg)
raise ValueError(error_msg)
if self.logger:
self.logger.info(
f"Successfully retrieved data for set {set_number}"
)
set_data = result["sets"][0]
our_set = {
"set_num": set_data["number"],
"set_name": set_data["name"],
"set_theme": set_data["theme"],
"set_image": set_data["image"]["thumbnailURL"],
"set_barcode": set_data["barcode"]["UPC"],
"set_retailprice": round(
set_data["LEGOCom"]["US"]["retailPrice"]
),
}
# Calculate adjusted price (10% markup)
retail_price = our_set["set_retailprice"]
our_set["set_adjustredprice"] = retail_price + (
(10 / 100) * retail_price
)
# Set PW inventory name with theme
our_set["pw_name"] = (
f"{our_set['set_num']} - {our_set['set_theme'].upper()} {our_set['set_name']}"
)
# Flag for manual price review if needed
if our_set["set_retailprice"] >= 110:
our_set["requires_manual_price"] = True
return our_set
elif response.status == 500:
error_msg = (
"Internal Server Error: Please check API key and parameters."
)
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
else:
error_msg = f"Unexpected response: {response.status}"
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
async def throttled_get_set(self, set_number: str) -> Dict[str, Any]:
"""Get set information with rate limiting"""
async with self.throttler:
return await self.get_set(set_number)
async def main():
"""Example usage"""
client = AsyncBricksetAPI(debug=True)
try:
set_info = await client.throttled_get_set("10349")
print(json.dumps(set_info, indent=2))
except Exception as e:
print(f"Error: {e}")
if __name__ == "__main__":
asyncio.run(main())
+23 -465
View File
@@ -4,15 +4,8 @@ import json
import asyncio
import uuid
import re
import traceback
from time import time
import aiohttp
from io import BytesIO
from square.core.api_error import ApiError
from . import db
from . import get_sq_env
import get_sq_env
def get_set_number(text: str) -> int:
@@ -55,462 +48,33 @@ async def get_itm(sku: str):
return {"name": name, "id": itm_id}
async def get_item_by_id(item_id: str) -> dict:
"""
Retrieve full item details by Square catalog item ID.
Uses batch_get API for direct ID lookup.
Args:
item_id: Square catalog item ID
Returns:
dict: Item details including name, category_id, variations, etc.
Raises:
ValueError: If item not found
Exception: If API error
"""
try:
# Use batch_get to retrieve item directly by ID
result = await client.catalog.batch_get(
object_ids=[item_id],
include_related_objects=False,
)
result_json = json.loads(result.model_dump_json())
if not result_json.get("objects"):
raise ValueError(f"Item {item_id} not found (may have been deleted)")
obj = result_json["objects"][0]
item_data = obj.get("item_data", {})
return {
"id": obj["id"],
"item_data": item_data, # Return the entire item_data object
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
}
except ValueError:
raise
except Exception as e:
print(f"[@ERROR] Failed to get item {item_id}: {e}")
raise
async def get_item_by_variation_id(variation_id: str) -> dict:
"""
Retrieve parent item details from a variation ID.
Webhooks send variation IDs (inventory.count.updated), but we need
the parent item to extract the set number from the name.
Since async Square API lacks direct object retrieval, we search all items
in the LEGO category and match variations.
Args:
variation_id: Square catalog item variation ID
Returns:
dict: Parent item details
Raises:
Exception: If variation/item not found or API error
"""
try:
# Search LEGO category items to find which one has this variation
print(f"[@DEBUG] Searching for variation {variation_id} in LEGO category")
cursor = None
page_count = 0
max_pages = 50 # Safety limit to prevent infinite loops
while page_count < max_pages:
page_count += 1
print(f"[@DEBUG] Searching page {page_count}...")
search_params = {
"object_types": ["ITEM"],
"query": {
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
"limit": 1000,
}
if cursor:
search_params["cursor"] = cursor
result = await client.catalog.search(**search_params)
result_json = json.loads(result.model_dump_json())
if not result_json.get("objects"):
if page_count == 1:
raise ValueError(f"No items found in LEGO category")
else:
# No more results
print(f"[@DEBUG] No more results on page {page_count}")
break
# Search through items on this page to find which has this variation
for obj in result_json["objects"]:
item_data = obj.get("item_data", {})
variations = item_data.get("variations", [])
for variation in variations:
if variation.get("id") == variation_id:
# Found the parent item!
print(
f"[@DEBUG] Found parent item: {obj['id']} on page {page_count}"
)
# Since we searched by category, we know it's in LEGO_CAT_ID
# But also try to get from item_data as fallback
category_id = item_data.get("category_id") or LEGO_CAT_ID
print(
f"[@DEBUG] Item category_id from data: {item_data.get('category_id')}"
)
print(f"[@DEBUG] Using category_id: {category_id}")
return {
"id": obj["id"],
"name": item_data.get("name", ""),
"description": item_data.get("description", ""),
"category_id": category_id,
"variations": variations,
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
}
# Check if there are more pages
cursor = result_json.get("cursor")
if not cursor:
# No more pages
print(f"[@DEBUG] No more pages after page {page_count}")
break
# If we get here, variation wasn't found
if page_count >= max_pages:
raise ValueError(
f"Variation {variation_id} not found after searching {max_pages} pages (possible infinite loop prevented)"
)
raise ValueError(
f"Variation {variation_id} not found in any LEGO category item (searched {page_count} pages)"
)
except Exception as e:
print(f"[@ERROR] Failed to get item from variation {variation_id}: {e}")
raise
async def upload_item_image(item_id: str, image_url: str) -> bool:
"""
Download image from URL and upload to Square catalog item.
Args:
item_id: Square catalog item ID
image_url: URL to download image from (e.g., Brickset thumbnail)
Returns:
bool: True if upload successful, False otherwise
"""
try:
print(f"[@DEBUG] Downloading image from {image_url}")
# Download image from URL
async with aiohttp.ClientSession() as session:
async with session.get(image_url) as resp:
if resp.status != 200:
print(f"[@ERROR] Failed to download image: HTTP {resp.status}")
return False
image_data = await resp.read()
# Determine content type from URL extension
content_type = "image/jpeg"
if image_url.lower().endswith(".png"):
content_type = "image/png"
elif image_url.lower().endswith(".gif"):
content_type = "image/gif"
print(
f"[@DEBUG] Uploading {len(image_data)} bytes as {content_type} to item {item_id}"
)
# Upload to Square
result = await client.catalog.images.create(
image_file=image_data,
request={
"idempotency_key": str(uuid.uuid4()),
"image": {
"type": "IMAGE",
"id": f"#IMAGE_{item_id}",
"image_data": {
"name": f"LEGO Set Image",
},
},
"object_id": item_id,
},
image_file_name=f"lego_{item_id}.jpg",
content_type=content_type,
)
result_json = json.loads(result.model_dump_json())
if result_json.get("image"):
print(f"[@DEBUG] ✓ Successfully uploaded image for item {item_id}")
return True
else:
print(f"[@ERROR] Image upload failed: {result_json}")
return False
except Exception as e:
print(f"[@ERROR] Failed to upload image for item {item_id}: {e}")
return False
async def update_itm(item_id: str, lego_set: dict, item_data: dict = None):
async def update_itm(lego_set: dict):
"""
Asynchronously updates an item in the Square catalog based on the provided LEGO set information.
Args:
item_id (str): Square catalog item ID to update
lego_set (dict): A dictionary containing LEGO set details from abricksetapi. Expected keys include:
- "set_num": LEGO set number (e.g., "75192")
- "set_name": Official LEGO set name
- "set_theme": LEGO theme (e.g., "Star Wars")
- "set_barcode": UPC barcode
- "set_image": Thumbnail URL
- "pw_name": Formatted name for Square inventory (e.g., "75192 - STAR WARS UCS Millennium Falcon")
- "set_adjustredprice": Retail price with 10% markup applied
item_data (dict): Optional pre-fetched item data to avoid redundant API call
lego_set (dict): A dictionary containing LEGO set details. Expected keys include:
- "sets": List of set dictionaries, each with "name", "number", "barcode" (with "UPC"), and "image" (with "thumbnailURL").
- "pw_name": The name associated with the LEGO set in another system.
- "set_adjustredprice": The adjusted price for the set.
Returns:
bool: True if the update operation was successful.
Raises:
KeyError: If expected keys are missing from the input dictionary.
Exception: For errors during Square API interaction.
"""
try:
set_number = lego_set["set_num"]
set_name = lego_set["set_name"]
set_theme = lego_set["set_theme"]
set_upc = lego_set["set_barcode"]
set_image = lego_set["set_image"]
pw_name = lego_set["pw_name"]
set_adjustedprice = lego_set["set_adjustredprice"]
price_in_cents = int(set_adjustedprice * 100)
set_name = lego_set["sets"][0]["name"]
set_number = lego_set["sets"][0]["number"]
set_upc = lego_set["sets"][0]["barcode"]["UPC"]
set_image = lego_set["sets"][0]["image"]["thumbnailURL"]
pw_name = lego_set["pw_name"]
set_adjustedprice = lego_set["set_adjustredprice"]
# Retry logic for version conflicts (race conditions with concurrent updates)
max_retries = 3
retry_delay = 0.5 # seconds
result = await client.catalog.search_items(text_filter=set_number, limit=1)
itm_id = json.loads(result.model_dump_json())["items"][0]["id"]
for attempt in range(max_retries):
try:
# ALWAYS fetch fresh item data right before update to get latest version
print(
f"[@DEBUG] Fetching fresh item data (attempt {attempt + 1}/{max_retries})..."
)
fresh_item = await get_item_by_id(item_id)
except ValueError as e:
# Item was deleted between webhook and enrichment
print(f"[@WARNING] {e} - skipping update (item may have been deleted)")
return
variations = fresh_item.get("item_data", {}).get("variations", [])
if not variations:
print(
f"[@WARNING] Item {item_id} has no variations, creating new variation"
)
variation_id = None
variation_version = None
present_at_location_ids = []
present_at_all_locations = False
else:
variation_id = variations[0]["id"]
variation_version = variations[0].get("version")
# Check if variation is at all locations
present_at_all_locations = variations[0].get(
"present_at_all_locations", False
)
# Extract location IDs from variation (only if not at all locations)
present_at_location_ids = variations[0].get(
"present_at_location_ids", []
)
# Get current version for version control
current_version = fresh_item.get("version")
# CRITICAL: batch_get NEVER returns category_id (always None)
# We MUST use the category from initial item_data (from search which includes category)
# or fallback to LEGO_CAT_ID since we know this is a LEGO item
if item_data and item_data.get("category_id"):
category_id = item_data.get("category_id")
print(f"[@DEBUG] Using category_id from item_data: {category_id}")
else:
# Fallback to LEGO_CAT_ID (we know it's a LEGO item if enrichment was triggered)
category_id = LEGO_CAT_ID
print(
f"[@WARNING] No category_id in item_data, using LEGO_CAT_ID: {LEGO_CAT_ID}"
)
print(f"[@DEBUG] Fresh item version: {current_version}")
print(f"[@DEBUG] Fresh variation version: {variation_version}")
print(f"[@DEBUG] Final category_id for upsert: {category_id}")
# Get the full existing item_data and modify only what we need
# This preserves ALL existing fields (location settings, tax, etc)
existing_item_data = fresh_item.get("item_data", {})
# Update item-level fields
existing_item_data["name"] = pw_name
existing_item_data["description"] = f"{set_name} - {set_theme}"
existing_item_data["category_id"] = category_id
# Update only the specific variation, keep all others unchanged
existing_variations = existing_item_data.get("variations", [])
updated_variations = []
variation_found = False
for existing_var in existing_variations:
if existing_var.get("id") == variation_id:
# Preserve all variation fields and only update what we need
updated_var = existing_var.copy()
updated_var_data = existing_var.get(
"item_variation_data", {}
).copy()
# Check if this variation has VARIABLE_PRICING
current_pricing_type = updated_var_data.get("pricing_type")
# Only set price if not using variable pricing
if current_pricing_type != "VARIABLE_PRICING":
updated_var_data["price_money"] = {
"amount": price_in_cents,
"currency": "USD",
}
print(f"[@DEBUG] Setting price: ${set_adjustedprice} for variation {variation_id}")
else:
print(f"[@WARNING] Skipping price update for {variation_id} - has VARIABLE_PRICING enabled")
updated_var_data["sku"] = set_number # LEGO set number
updated_var_data["upc"] = (
set_upc if set_upc else None
) # Barcode/GTIN
updated_var["item_variation_data"] = updated_var_data
# Update version (required for updates)
if variation_version is not None:
updated_var["version"] = variation_version
updated_variations.append(updated_var)
variation_found = True
else:
# Keeprice_money": {
"amount": price_in_cents,
"currency": "USD",
},
"p other variations as-is
updated_variations.append(existing_var)
# If variation wasn't found (shouldn't happen), create new one
if not variation_found:
new_variation = {
"type": "ITEM_VARIATION",
"id": variation_id if variation_id else "#TEMP_VARIATION_ID",
"item_variation_data": {
"item_id": item_id,
"name": "Regular",
"pricing_type": "FIXED_PRICING",
"sku": set_number,
"upc": set_upc if set_upc else None,
},
}
if variation_version is not None:
new_variation["version"] = variation_version
updated_variations.append(new_variation)
existing_item_data["variations"] = updated_variations
item_obj = {
"type": "ITEM",
"id": item_id,
"version": current_version,
"item_data": existing_item_data,
}
upsert_body = {
"idempotency_key": str(uuid.uuid4()),
"object": item_obj,
}
# Remove None values
if not set_upc:
del variation_obj["item_variation_data"]["sku"]
print(f"[@DEBUG] Upserting item {item_id} with name: {pw_name}")
print(f"[@DEBUG] Upsert body: {json.dumps(upsert_body, indent=2)}")
# Perform upsert using catalog.object.upsert
try:
result = await client.catalog.object.upsert(
idempotency_key=upsert_body["idempotency_key"],
object=upsert_body["object"],
)
result_json = json.loads(result.model_dump_json())
if result_json.get("catalog_object"):
print(f"[@DEBUG] ✓ Successfully updated item {item_id}")
# DEBUG: Check what Square actually returned
returned_item = result_json["catalog_object"]
returned_category = returned_item.get("item_data", {}).get(
"category_id"
)
print(f"[@DEBUG] Square returned category_id: {returned_category}")
# Upload image if available (DISABLED - concerns about storage costs)
# if set_image:
# print(f"[@DEBUG] Uploading image from {set_image}")
# await upload_item_image(item_id, set_image)
return True
else:
print(
f"[@ERROR] Failed to update item {item_id}: No catalog_object in response"
)
return False
except ApiError as e:
# Check if this is a VERSION_MISMATCH error
if e.status_code == 400 and e.body and isinstance(e.body, dict):
errors = e.body.get("errors", [])
if any(err.get("code") == "VERSION_MISMATCH" for err in errors):
if attempt < max_retries - 1:
print(
f"[@WARNING] Version conflict on attempt {attempt + 1}, retrying in {retry_delay}s..."
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
continue # Retry with fresh data
else:
print(
f"[@ERROR] Version conflict persists after {max_retries} attempts, giving up"
)
return False
# Re-raise other API errors
raise
except Exception as e:
print(f"[@ERROR] Failed to update item {item_id}: {e}")
traceback.print_exc()
return False
# await sq.catalog.object.upsert()
return True
async def acat2id(cat_id: str):
@@ -539,17 +103,7 @@ async def acat2id(cat_id: str):
{
"id": obj["id"],
"name": obj.get("item_data", {}).get("name", "Unknown"),
"number": (
None
if not obj.get("item_data", {}).get("name", "")
else (
get_set_number(obj.get("item_data", {}).get("name", ""))
if re.findall(r"\d{5}", obj.get("item_data", {}).get("name", ""))
else None
)
),
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
"number": get_set_number(obj.get("item_data", {}).get("name", ""))
}
for obj in result_json.get("objects", [])
]
@@ -558,10 +112,14 @@ async def acat2id(cat_id: str):
async def test() -> None:
result = await acat2id(LEGO_CAT_ID) # type: ignore
task = asyncio.create_task(acat2id(LEGO_CAT_ID)) # type: ignore
await asyncio.sleep(1) # Give the task some time to complete
result = await task
print("OBJECTS IN LEGO CATEGORY:")
for item in result:
print(f"ID: {item['id']}, Name: {item['name']}, Updated: {item['updated_at']}")
print(f"ID: {item['id']}, Name: {item['name']}")
# eturn
# pass
"""
@@ -1,70 +1,60 @@
import os
import requests
import json
import dotenv
dotenv.load_dotenv()
USERNAME = os.getenv("BRICKSET_USERNAME")
PASS = os.getenv("BRICKSET_PASS")
BRICKSET_API_KEY = os.getenv("BRICKSET_API_KEY")
debug = False # Set to True to enable debug output
def login(username, password):
url = "https://brickset.com/api/v3.asmx/login"
headers = {"UserAgent": "studspider/0.1"}
params = {
"UserAgent": "studspider/0.1",
"username": username,
"password": password,
"apiKey": BRICKSET_API_KEY,
}
response = requests.get(url, params=params, headers=headers)
if debug:
print(f"[@DEBUG] Login response: {response.status_code} - {response.text}\n")
return response.json()
def get_set(set_number: int) -> dict:
login_response = login(USERNAME, PASS)
url = f"https://brickset.com/api/v3.asmx/getSets?apiKey={BRICKSET_API_KEY}&userHash={login_response["hash"]}&params={{'setNumber' : '{set_number}-1'}}"
response = requests.get(url)
if response.status_code == 200:
data = response.json()
if not data["sets"] and data["message"] == "API limit exceeded":
raise Exception("API limit exceeded. Please try again later.")
return False
print(f"[@DEBUG]\t{data}")
# Try to get UPC barcode, fall back to EAN if not available
barcode_data = data["sets"][0].get("barcode", {})
upc_code = barcode_data.get("UPC") or barcode_data.get("EAN")
our_set = {
"set_num": data["sets"][0]["number"],
"set_name": data["sets"][0]["name"],
"set_theme": data["sets"][0]["theme"],
"set_image": data["sets"][0]["image"]["thumbnailURL"],
"set_barcode": upc_code,
"set_retailprice": round(data["sets"][0]["LEGOCom"]["US"]["retailPrice"]),
}
# add PW 10% b4 tax
our_set.update(
{"set_adjustredprice": our_set + ((10 / 100) * our_set["set_retailprice"])}
)
our_set["pw_name"] = (
f"{our_set['set_num']} - {our_set["set_theme"].upper()} - {our_set['set_name']}"
)
if our_set["set_retailprice"] >= 110:
our_set["requires_manual_price"] = True
return our_set
elif response.status_code == 500:
raise Exception(
"Internal Server Error: Please check the API key and parameters."
)
else:
raise Exception("Unexpected response from the API.")
import os
import requests
import json
import dotenv
dotenv.load_dotenv()
USERNAME = os.getenv("BRICKSET_USERNAME")
PASS = os.getenv("BRICKSET_PASS")
BRICKSET_API_KEY = os.getenv("BRICKSET_API_KEY")
debug = False # Set to True to enable debug output
def login(username, password):
url = "https://brickset.com/api/v3.asmx/login"
headers = {"UserAgent": "Edderkop/0.1"}
params = {
"UserAgent": "Edderkop/0.1",
"username": username,
"password": password,
"apiKey": BRICKSET_API_KEY,
}
response = requests.get(url, params=params, headers=headers)
if debug:
print(f"[@DEBUG] Login response: {response.status_code} - {response.text}\n")
return response.json()
def get_set(set_number: int) -> dict:
login_response = login(USERNAME, PASS)
url = f"https://brickset.com/api/v3.asmx/getSets?apiKey={BRICKSET_API_KEY}&userHash={login_response["hash"]}&params={{'setNumber' : '{set_number}-1'}}"
response = requests.get(url)
if response.status_code == 200:
data = response.json()
our_set = {
"set_num": data["sets"][0]["number"],
"set_name": data["sets"][0]["name"],
"set_theme": data["sets"][0]["theme"],
"set_image": data["sets"][0]["image"]["thumbnailURL"],
"set_barcode": data["sets"][0]["barcode"]["UPC"],
"set_retailprice": round(data["sets"][0]["LEGOCom"]["US"]["retailPrice"]),
}
# add PW 10% b4 tax
our_set.update(
{"set_adjustredprice": our_set + ((10 / 100) * our_set["set_retailprice"])}
)
our_set["pw_name"] = (
f"{our_set['set_num']} - {our_set["set_theme"].upper()} {our_set['set_name']}"
)
if our_set["set_retailprice"] >= 110:
our_set["requires_manual_price"] = True
return our_set
elif response.status_code == 500:
raise Exception(
"Internal Server Error: Please check the API key and parameters."
)
else:
raise Exception("Unexpected response from the API.")
-61
View File
@@ -1,61 +0,0 @@
import aiosqlite
from time import time
time = int(time()) # we dont need no stinkin floats
# db = await aiosqlite.connect("studspider.db") # type: ignore
async def close(db):
await db.close()
async def fetch(table, sqobj):
"""
Asynchronously fetch a record from the REVIEW table by sq_object.
Args:
sqobj (str): The sq_object identifier to search for.
Returns:
dict or None: The fetched record as a dictionary, or None if not found.
"""
async with aiosqlite.connect("studspider.db") as db: # type: ignore
async with db.execute(
f"SELECT * FROM {table} WHERE sq_object = {sqobj}"
) as cursor:
async for row in cursor:
return row
return None
async def upsert(table: str, data: dict, unique_key: str) -> bool:
"""
Asynchronously insert or update a record in the specified table.
Args:
table (str): The name of the table to upsert into.
data (dict): A dictionary of column-value pairs to insert or update.
unique_key (str): The column name that serves as the unique key for upsert.
Returns:
None
"""
print(f"Upserting into {table}: {data}")
columns = ", ".join(data.keys())
placeholders = ", ".join("?" * len(data))
updates = ", ".join(
f"{col}=excluded.{col}" for col in data.keys() if col != unique_key
)
sql = f"""
INSERT INTO {table} ({columns})
VALUES ({placeholders})
ON CONFLICT({unique_key}) DO UPDATE SET
{updates};
"""
async with aiosqlite.connect("studspider.db") as db: # type: ignore
async with db.execute(sql, tuple(data.values())) as cursor:
await db.commit()
return True
+221 -221
View File
@@ -1,221 +1,221 @@
import aiosqlite
import logging
from typing import Optional, List, Dict, Any
import json
import os
logger = logging.getLogger(__name__)
# Get the absolute path to studspider.db in the project root
DEFAULT_DB_PATH = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "studspider.db"
)
if not os.path.exists(DEFAULT_DB_PATH):
raise FileNotFoundError(f"Database file not found at {DEFAULT_DB_PATH}")
print('Please run "src/scripts/studspider.SQL" to setup the database.')
os.exit(0)
class AsyncSchedulerDB:
"""
Asynchronous SQLite database manager for APScheduler.
Handles job storage, retrieval, and database operations using the main studspider.db database.
"""
def __init__(self, db_path: str = DEFAULT_DB_PATH):
"""
Initialize the database manager.
Args:
db_path (str, optional): Path to the SQLite database file.
Defaults to studspider.db in project root.
"""
self.db_path = db_path
self._initialized = False
logger.info(f"Using database at: {self.db_path}")
async def initialize(self):
"""Initialize the database schema if it doesn't exist."""
if self._initialized:
return
async with aiosqlite.connect(self.db_path) as db:
# Create apscheduler jobs table if it doesn't exist
await db.execute(
"""
CREATE TABLE IF NOT EXISTS apscheduler_jobs (
id VARCHAR(191) NOT NULL,
next_run_time FLOAT,
job_state BLOB NOT NULL,
PRIMARY KEY (id)
)
"""
)
await db.commit()
self._initialized = True
logger.info(f"Initialized scheduler database at {self.db_path}")
async def get_all_jobs(self) -> List[Dict[str, Any]]:
"""
Retrieve all jobs from the database.
Returns:
List[Dict[str, Any]]: List of job dictionaries
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"SELECT * FROM apscheduler_jobs ORDER BY next_run_time"
) as cursor:
rows = await cursor.fetchall()
return [
{
"id": row["id"],
"next_run_time": row["next_run_time"],
"job_state": json.loads(row["job_state"]),
}
for row in rows
]
async def get_job(self, job_id: str) -> Optional[Dict[str, Any]]:
"""
Retrieve a specific job by ID.
Args:
job_id (str): The ID of the job to retrieve
Returns:
Optional[Dict[str, Any]]: Job dictionary if found, None otherwise
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"SELECT * FROM apscheduler_jobs WHERE id = ?", (job_id,)
) as cursor:
row = await cursor.fetchone()
if row:
return {
"id": row["id"],
"next_run_time": row["next_run_time"],
"job_state": json.loads(row["job_state"]),
}
return None
async def add_job(
self, job_id: str, next_run_time: float, job_state: Dict[str, Any]
) -> bool:
"""
Add a new job to the database.
Args:
job_id (str): Unique identifier for the job
next_run_time (float): Unix timestamp for next scheduled run
job_state (Dict[str, Any]): Job state dictionary
Returns:
bool: True if successful, False otherwise
"""
await self.initialize()
try:
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"INSERT INTO apscheduler_jobs (id, next_run_time, job_state) VALUES (?, ?, ?)",
(job_id, next_run_time, json.dumps(job_state)),
)
await db.commit()
return True
except aiosqlite.IntegrityError:
logger.error(f"Job with ID {job_id} already exists")
return False
except Exception as e:
logger.error(f"Error adding job: {e}")
return False
async def update_job(
self, job_id: str, next_run_time: float, job_state: Dict[str, Any]
) -> bool:
"""
Update an existing job in the database.
Args:
job_id (str): ID of the job to update
next_run_time (float): New next run time
job_state (Dict[str, Any]): Updated job state
Returns:
bool: True if successful, False if job not found
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"""UPDATE apscheduler_jobs
SET next_run_time = ?, job_state = ?
WHERE id = ?""",
(next_run_time, json.dumps(job_state), job_id),
)
await db.commit()
return cursor.rowcount > 0
async def remove_job(self, job_id: str) -> bool:
"""
Remove a job from the database.
Args:
job_id (str): ID of the job to remove
Returns:
bool: True if job was removed, False if not found
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"DELETE FROM apscheduler_jobs WHERE id = ?", (job_id,)
)
await db.commit()
return cursor.rowcount > 0
async def remove_all_jobs(self) -> int:
"""
Remove all jobs from the database.
Returns:
int: Number of jobs removed
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute("DELETE FROM apscheduler_jobs")
await db.commit()
return cursor.rowcount
async def get_due_jobs(self, now: float) -> List[Dict[str, Any]]:
"""
Get all jobs that are due to run.
Args:
now (float): Current timestamp
Returns:
List[Dict[str, Any]]: List of due jobs
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"""SELECT * FROM apscheduler_jobs
WHERE next_run_time <= ?
ORDER BY next_run_time""",
(now,),
) as cursor:
rows = await cursor.fetchall()
return [
{
"id": row["id"],
"next_run_time": row["next_run_time"],
"job_state": json.loads(row["job_state"]),
}
for row in rows
]
import aiosqlite
import logging
from typing import Optional, List, Dict, Any
import json
import os
logger = logging.getLogger(__name__)
# Get the absolute path to edderkop.db in the project root
DEFAULT_DB_PATH = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "edderkop.db"
)
if not os.path.exists(DEFAULT_DB_PATH):
raise FileNotFoundError(f"Database file not found at {DEFAULT_DB_PATH}")
print('Please run "src/scripts/EDDERKOP.SQL" to setup the database.')
os.exit(0)
class AsyncSchedulerDB:
"""
Asynchronous SQLite database manager for APScheduler.
Handles job storage, retrieval, and database operations using the main edderkop.db database.
"""
def __init__(self, db_path: str = DEFAULT_DB_PATH):
"""
Initialize the database manager.
Args:
db_path (str, optional): Path to the SQLite database file.
Defaults to edderkop.db in project root.
"""
self.db_path = db_path
self._initialized = False
logger.info(f"Using database at: {self.db_path}")
async def initialize(self):
"""Initialize the database schema if it doesn't exist."""
if self._initialized:
return
async with aiosqlite.connect(self.db_path) as db:
# Create apscheduler jobs table if it doesn't exist
await db.execute(
"""
CREATE TABLE IF NOT EXISTS apscheduler_jobs (
id VARCHAR(191) NOT NULL,
next_run_time FLOAT,
job_state BLOB NOT NULL,
PRIMARY KEY (id)
)
"""
)
await db.commit()
self._initialized = True
logger.info(f"Initialized scheduler database at {self.db_path}")
async def get_all_jobs(self) -> List[Dict[str, Any]]:
"""
Retrieve all jobs from the database.
Returns:
List[Dict[str, Any]]: List of job dictionaries
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"SELECT * FROM apscheduler_jobs ORDER BY next_run_time"
) as cursor:
rows = await cursor.fetchall()
return [
{
"id": row["id"],
"next_run_time": row["next_run_time"],
"job_state": json.loads(row["job_state"]),
}
for row in rows
]
async def get_job(self, job_id: str) -> Optional[Dict[str, Any]]:
"""
Retrieve a specific job by ID.
Args:
job_id (str): The ID of the job to retrieve
Returns:
Optional[Dict[str, Any]]: Job dictionary if found, None otherwise
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"SELECT * FROM apscheduler_jobs WHERE id = ?", (job_id,)
) as cursor:
row = await cursor.fetchone()
if row:
return {
"id": row["id"],
"next_run_time": row["next_run_time"],
"job_state": json.loads(row["job_state"]),
}
return None
async def add_job(
self, job_id: str, next_run_time: float, job_state: Dict[str, Any]
) -> bool:
"""
Add a new job to the database.
Args:
job_id (str): Unique identifier for the job
next_run_time (float): Unix timestamp for next scheduled run
job_state (Dict[str, Any]): Job state dictionary
Returns:
bool: True if successful, False otherwise
"""
await self.initialize()
try:
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"INSERT INTO apscheduler_jobs (id, next_run_time, job_state) VALUES (?, ?, ?)",
(job_id, next_run_time, json.dumps(job_state)),
)
await db.commit()
return True
except aiosqlite.IntegrityError:
logger.error(f"Job with ID {job_id} already exists")
return False
except Exception as e:
logger.error(f"Error adding job: {e}")
return False
async def update_job(
self, job_id: str, next_run_time: float, job_state: Dict[str, Any]
) -> bool:
"""
Update an existing job in the database.
Args:
job_id (str): ID of the job to update
next_run_time (float): New next run time
job_state (Dict[str, Any]): Updated job state
Returns:
bool: True if successful, False if job not found
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"""UPDATE apscheduler_jobs
SET next_run_time = ?, job_state = ?
WHERE id = ?""",
(next_run_time, json.dumps(job_state), job_id),
)
await db.commit()
return cursor.rowcount > 0
async def remove_job(self, job_id: str) -> bool:
"""
Remove a job from the database.
Args:
job_id (str): ID of the job to remove
Returns:
bool: True if job was removed, False if not found
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute(
"DELETE FROM apscheduler_jobs WHERE id = ?", (job_id,)
)
await db.commit()
return cursor.rowcount > 0
async def remove_all_jobs(self) -> int:
"""
Remove all jobs from the database.
Returns:
int: Number of jobs removed
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
cursor = await db.execute("DELETE FROM apscheduler_jobs")
await db.commit()
return cursor.rowcount
async def get_due_jobs(self, now: float) -> List[Dict[str, Any]]:
"""
Get all jobs that are due to run.
Args:
now (float): Current timestamp
Returns:
List[Dict[str, Any]]: List of due jobs
"""
await self.initialize()
async with aiosqlite.connect(self.db_path) as db:
db.row_factory = aiosqlite.Row
async with db.execute(
"""SELECT * FROM apscheduler_jobs
WHERE next_run_time <= ?
ORDER BY next_run_time""",
(now,),
) as cursor:
rows = await cursor.fetchall()
return [
{
"id": row["id"],
"next_run_time": row["next_run_time"],
"job_state": json.loads(row["job_state"]),
}
for row in rows
]
+116 -116
View File
@@ -1,116 +1,116 @@
import os
import requests
import json
import dotenv
from typing import Dict, Any
dotenv.load_dotenv()
class BricksetAPI:
"""Class for interacting with the Brickset API"""
def __init__(self, debug: bool = False):
self.username = os.getenv("BRICKSET_USERNAME")
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
self.debug = debug
self.headers = {"UserAgent": "studspider/0.1"}
self.user_hash = None
def login(self) -> Dict[str, Any]:
"""Login to Brickset API and get user hash"""
url = "https://brickset.com/api/v3.asmx/login"
params = {
"username": self.username,
"password": self.password,
"apiKey": self.api_key,
}
response = requests.get(url, params=params, headers=self.headers)
if self.debug:
print(f"[@DEBUG] Login response: {response.status_code}")
print(f"[@DEBUG] Response content: {response.text}")
try:
data = response.json()
self.user_hash = data.get("hash")
if not self.user_hash:
raise ValueError("No hash received in login response")
return data
except json.JSONDecodeError as e:
print(f"[@ERROR] Failed to decode JSON response: {e}")
print(f"[@DEBUG] Raw response: {response.text}")
raise
def get_set(self, set_number: str) -> Dict[str, Any]:
"""Get LEGO set information by set number"""
if not self.user_hash:
self.login()
url = "https://brickset.com/api/v3.asmx/getSets"
params = {
"apiKey": self.api_key,
"userHash": self.user_hash,
"params": json.dumps({"setNumber": f"{set_number}-1"}),
}
response = requests.get(url, params=params, headers=self.headers)
if self.debug:
print(f"[@DEBUG] Get set response: {response.status_code}")
print(f"[@DEBUG] Response content: {response.text}")
if response.status_code == 200:
try:
data = response.json()
if not data.get("sets"):
raise ValueError(f"No set data found for {set_number}")
set_data = data["sets"][0]
our_set = {
"set_num": set_data["number"],
"set_name": set_data["name"],
"set_image": set_data["image"]["thumbnailURL"],
"set_barcode": set_data["barcode"]["UPC"],
"set_retailprice": round(set_data["LEGOCom"]["US"]["retailPrice"]),
}
# Calculate adjusted price (10% markup)
retail_price = our_set["set_retailprice"]
our_set["set_adjustredprice"] = retail_price + (
(10 / 100) * retail_price
)
# Set Playwell inventory name
if our_set["theme"] == "Duplo":
our_set["pw_name"] = (
f"{our_set['set_num']} - DUPLO {our_set['set_name']}"
)
else:
our_set["pw_name"] = f"{our_set['set_num']} - {our_set['set_name']}"
return our_set
except json.JSONDecodeError as e:
# print(f"[@ERROR] Failed to decode JSON response: {e}")
# print(f"[@DEBUG] Raw response: {response.text}")
raise
except KeyError as e:
# print(f"[@ERROR] Missing expected field in response: {e}")
# print(f"[@DEBUG] Response data: {data}")
raise
elif response.status_code == 500:
raise Exception(
"Internal Server Error: Please check API key and parameters."
)
else:
raise Exception(f"Unexpected response: {response.status_code}")
def example():
client = BricksetAPI(debug=True)
try:
set_info = client.get_set("10931")
print(set_info)
except Exception as e:
print(f"Error: {e}")
import os
import requests
import json
import dotenv
from typing import Dict, Any
dotenv.load_dotenv()
class BricksetAPI:
"""Class for interacting with the Brickset API"""
def __init__(self, debug: bool = False):
self.username = os.getenv("BRICKSET_USERNAME")
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
self.debug = debug
self.headers = {"UserAgent": "Edderkop/0.1"}
self.user_hash = None
def login(self) -> Dict[str, Any]:
"""Login to Brickset API and get user hash"""
url = "https://brickset.com/api/v3.asmx/login"
params = {
"username": self.username,
"password": self.password,
"apiKey": self.api_key,
}
response = requests.get(url, params=params, headers=self.headers)
if self.debug:
print(f"[@DEBUG] Login response: {response.status_code}")
print(f"[@DEBUG] Response content: {response.text}")
try:
data = response.json()
self.user_hash = data.get("hash")
if not self.user_hash:
raise ValueError("No hash received in login response")
return data
except json.JSONDecodeError as e:
print(f"[@ERROR] Failed to decode JSON response: {e}")
print(f"[@DEBUG] Raw response: {response.text}")
raise
def get_set(self, set_number: str) -> Dict[str, Any]:
"""Get LEGO set information by set number"""
if not self.user_hash:
self.login()
url = "https://brickset.com/api/v3.asmx/getSets"
params = {
"apiKey": self.api_key,
"userHash": self.user_hash,
"params": json.dumps({"setNumber": f"{set_number}-1"}),
}
response = requests.get(url, params=params, headers=self.headers)
if self.debug:
print(f"[@DEBUG] Get set response: {response.status_code}")
print(f"[@DEBUG] Response content: {response.text}")
if response.status_code == 200:
try:
data = response.json()
if not data.get("sets"):
raise ValueError(f"No set data found for {set_number}")
set_data = data["sets"][0]
our_set = {
"set_num": set_data["number"],
"set_name": set_data["name"],
"set_image": set_data["image"]["thumbnailURL"],
"set_barcode": set_data["barcode"]["UPC"],
"set_retailprice": round(set_data["LEGOCom"]["US"]["retailPrice"]),
}
# Calculate adjusted price (10% markup)
retail_price = our_set["set_retailprice"]
our_set["set_adjustredprice"] = retail_price + (
(10 / 100) * retail_price
)
# Set Playwell inventory name
if our_set["theme"] == "Duplo":
our_set["pw_name"] = (
f"{our_set['set_num']} - DUPLO {our_set['set_name']}"
)
else:
our_set["pw_name"] = f"{our_set['set_num']} - {our_set['set_name']}"
return our_set
except json.JSONDecodeError as e:
# print(f"[@ERROR] Failed to decode JSON response: {e}")
# print(f"[@DEBUG] Raw response: {response.text}")
raise
except KeyError as e:
# print(f"[@ERROR] Missing expected field in response: {e}")
# print(f"[@DEBUG] Response data: {data}")
raise
elif response.status_code == 500:
raise Exception(
"Internal Server Error: Please check API key and parameters."
)
else:
raise Exception(f"Unexpected response: {response.status_code}")
def example():
client = BricksetAPI(debug=True)
try:
set_info = client.get_set("10931")
print(set_info)
except Exception as e:
print(f"Error: {e}")
+2 -15
View File
@@ -1,29 +1,16 @@
import os
import dotenv
from pathlib import Path
from square.environment import SquareEnvironment
# Load .env from project root (parent of src/)
env_path = Path(__file__).parent.parent / ".env"
dotenv.load_dotenv(env_path)
dotenv.load_dotenv()
LEGO_CAT_ID = os.getenv("SQUARE_LEGO_CAT_ID")
SQUARE_WEBHOOK_URL = os.getenv("SQUARE_WEBHOOK_URL")
# Check environment mode
IS_PRODUCTION = os.getenv("SQUARE_ENV") == "production"
DEBUG_MODE = not IS_PRODUCTION # Debug logging enabled in sandbox/dev
if IS_PRODUCTION:
if os.getenv("SQUARE_ENV") == "production":
SQUARE_APP_ID = os.getenv("PROD_SQUARE_APP_ID")
SQUARE_TOKEN = os.getenv("PROD_SQUARE_TOKEN")
SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("PROD_SQUARE_WEBHOOK_SIGNATURE_KEY")
SQUARE_WEBHOOK_SUB_ID = os.getenv("PROD_SQUARE_WEBHOOK_SUB_ID")
environment = SquareEnvironment.PRODUCTION
else:
SQUARE_APP_ID = os.getenv("SANDBOX_SQUARE_APP_ID")
SQUARE_TOKEN = os.getenv("SANDBOX_SQUARE_TOKEN")
SQUARE_WEBHOOK_SIGNATURE_KEY = os.getenv("SANDBOX_SQUARE_WEBHOOK_SIGNATURE_KEY")
SQUARE_WEBHOOK_SUB_ID = os.getenv("SANDBOX_SQUARE_WEBHOOK_SUB_ID")
environment = SquareEnvironment.SANDBOX
-217
View File
@@ -1,217 +0,0 @@
import asyncio
import json
import re
from . import get_sq_env
import aiosqlite
from time import time
from . import asquare
from . import db
from . import abricksetapi
from . import throttler
Throttler = throttler.GlobalThrottler
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
async def enrich_single_item(item_id: str, item_data: dict = None, logger=None) -> bool:
r"""
Enrich a single Square catalog item with Brickset data.
Flow:
1. Get item details from Square (by item_id) - skip if item_data provided
2. Extract 5-digit set number from name (regex \d{5})
3. Fetch Brickset data (with throttling)
4. Update Square item with enriched data
5. Handle errors (no set number → MISSING table)
Args:
item_id: Square catalog item ID
item_data: Optional pre-fetched item data (avoids redundant API call)
logger: Optional logger instance for debug output
Returns:
True if successful, False otherwise
"""
try:
if logger:
logger.info(f"Starting enrichment for item {item_id}")
else:
print(f"[@DEBUG] Starting enrichment for item {item_id}")
# Get item details from Square (only if not already provided)
if item_data is None:
item = await asquare.get_item_by_id(item_id)
else:
item = item_data
item_name = item.get("name", "")
if logger:
logger.debug(f"Item name: {item_name}")
else:
print(f"[@DEBUG] Item name: {item_name}")
# Extract 5-digit set number
set_numbers = re.findall(r"\d{5}", item_name)
if not set_numbers:
# No set number found - log to MISSING table
if logger:
logger.warning(f"No set number found in item {item_id}: {item_name}")
else:
print(f"[@WARNING] No set number found in item {item_id}: {item_name}")
await db.upsert(
"MISSING",
{
"sq_object": item_id,
"time_added": int(time()),
"set_number": None,
"set_name": item_name,
},
"sq_object",
)
return False
set_number = set_numbers[0]
# Check if item is already enriched (format: "XXXXX - THEME Name")
# Pattern: 5 digits, space, dash, space, uppercase theme
if re.match(r"^\d{5}\s-\s[A-Z]", item_name):
if logger:
logger.info(f"Item {item_id} already enriched, skipping: {item_name}")
else:
print(
f"[@DEBUG] Item {item_id} already enriched, skipping: {item_name}"
)
return True
if logger:
logger.info(f"Extracted set number: {set_number}")
else:
print(f"[@DEBUG] Extracted set number: {set_number}")
# Fetch Brickset data (with throttling)
# debug flag will be auto-set based on SQUARE_ENV
brickset_client = abricksetapi.AsyncBricksetAPI(logger=logger)
lego_set = await brickset_client.throttled_get_set(set_number)
if logger:
logger.info(f"Retrieved Brickset data for {lego_set['pw_name']}")
else:
print(f"[@DEBUG] Retrieved Brickset data for {lego_set['pw_name']}")
# Note: Price updates disabled - enriching without price
# REVIEW table logic skipped
# Update Square item with enriched data (without price)
success = await asquare.update_itm(item_id, lego_set, item_data=item)
if success:
if logger:
logger.info(f"✓ Successfully enriched item {item_id}")
else:
print(f"[@DEBUG] ✓ Successfully enriched item {item_id}")
return True
else:
if logger:
logger.error(f"Failed to update Square item {item_id}")
else:
print(f"[@ERROR] Failed to update Square item {item_id}")
return False
except ValueError as e:
if logger:
logger.error(f"Value error enriching item {item_id}: {e}")
else:
print(f"[@ERROR] Value error enriching item {item_id}: {e}")
return False
except Exception as e:
if logger:
logger.error(f"Unexpected error enriching item {item_id}: {e}")
else:
print(f"[@ERROR] Unexpected error enriching item {item_id}: {e}")
import traceback
traceback.print_exc()
return False
async def enrich_bulk(logger=None) -> None:
"""
Enrich all items in the LEGO category (bulk operation).
This is the refactored RUNUPDATE command logic.
Iterates all items in LEGO category and enriches each one.
Args:
logger: Optional logger instance for debug output
"""
try:
if logger:
logger.info("Starting bulk enrichment for LEGO category")
else:
print(f"[@DEBUG] Starting bulk enrichment for LEGO category {LEGO_CAT_ID}")
# Get all items in LEGO category
items = await asquare.acat2id(LEGO_CAT_ID)
if logger:
logger.info(f"Found {len(items)} items in LEGO category")
else:
print(f"[@DEBUG] Found {len(items)} items in LEGO category")
# Enrich each item with concurrent batches
successful = 0
failed = 0
batch_size = 20 # Process 20 items concurrently
for i in range(0, len(items), batch_size):
batch = items[i : i + batch_size]
batch_num = (i // batch_size) + 1
total_batches = (len(items) + batch_size - 1) // batch_size
if logger:
logger.info(
f"Processing batch {batch_num}/{total_batches} ({len(batch)} items)"
)
else:
print(
f"[@DEBUG] Processing batch {batch_num}/{total_batches} ({len(batch)} items)"
)
# Process batch concurrently
tasks = [enrich_single_item(item["id"], logger=logger) for item in batch]
results = await asyncio.gather(*tasks, return_exceptions=True)
# Count results
for result in results:
if isinstance(result, Exception):
failed += 1
elif result:
successful += 1
else:
failed += 1
if logger:
logger.info(
f"Bulk enrichment complete: {successful} successful, {failed} failed"
)
else:
print(
f"[@DEBUG] Bulk enrichment complete: {successful} successful, {failed} failed"
)
except Exception as e:
if logger:
logger.error(f"Error during bulk enrichment: {e}")
else:
print(f"[@ERROR] Error during bulk enrichment: {e}")
import traceback
traceback.print_exc()
@@ -1,66 +1,66 @@
import json
import get_sq_env
SQUARE_TOKEN = get_sq_env.SQUARE_TOKEN
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
ENVIRONMENT = get_sq_env.environment
from square import Square
from square.environment import SquareEnvironment
def get_square_client():
"""Initialize and return a Square client."""
if not SQUARE_TOKEN:
raise ValueError("SQUARE_TOKEN environment variable is not set.")
square = Square(token=SQUARE_TOKEN, environment=ENVIRONMENT) # type: ignore
return square
client = get_square_client()
def get_itm(item_id: str):
"""Fetch an item from Square by its ID."""
result = client.catalog.search_items(text_filter=item_id, limit=1)
result_json = json.loads(result.model_dump_json())
ids = {
"item_id": result_json["items"][0]["id"],
"vari_id": result_json["items"][0]["item_data"]["variations"][0][
"item_variation_data"
]["item_id"],
}
item = {
"set_name": result_json["items"][0]["item_data"]["name"],
"set_price": result_json["items"][0]["item_data"]["variations"][0][
"item_variation_data"
]["price_money"],
"set_barcode": result_json["items"][0]["item_data"]["variations"][0][
"item_variation_data"
]["sku"],
}
return [ids, item]
def cat2id(cat_id: str):
"""Return a list of item IDs within the specified category ID."""
result = client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": cat_id,
}
},
)
result_json = json.loads(result.model_dump_json())
item_ids = [obj["id"] for obj in result_json.get("objects", [])]
return item_ids
def test() -> None:
# all_lego_sets = cat2id(SQUARE_LEGO_CAT_ID) # type: ignore
# result = get_itm("71433")
# print(result)
pass
import json
import get_sq_env
SQUARE_TOKEN = get_sq_env.SQUARE_TOKEN
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
ENVIRONMENT = get_sq_env.environment
from square import Square
from square.environment import SquareEnvironment
def get_square_client():
"""Initialize and return a Square client."""
if not SQUARE_TOKEN:
raise ValueError("SQUARE_TOKEN environment variable is not set.")
square = Square(token=SQUARE_TOKEN, environment=ENVIRONMENT) # type: ignore
return square
client = get_square_client()
def get_itm(item_id: str):
"""Fetch an item from Square by its ID."""
result = client.catalog.search_items(text_filter=item_id, limit=1)
result_json = json.loads(result.model_dump_json())
ids = {
"item_id": result_json["items"][0]["id"],
"vari_id": result_json["items"][0]["item_data"]["variations"][0][
"item_variation_data"
]["item_id"],
}
item = {
"set_name": result_json["items"][0]["item_data"]["name"],
"set_price": result_json["items"][0]["item_data"]["variations"][0][
"item_variation_data"
]["price_money"],
"set_barcode": result_json["items"][0]["item_data"]["variations"][0][
"item_variation_data"
]["sku"],
}
return [ids, item]
def cat2id(cat_id: str):
"""Return a list of item IDs within the specified category ID."""
result = client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": cat_id,
}
},
)
result_json = json.loads(result.model_dump_json())
item_ids = [obj["id"] for obj in result_json.get("objects", [])]
return item_ids
def test() -> None:
# all_lego_sets = cat2id(SQUARE_LEGO_CAT_ID) # type: ignore
# result = get_itm("71433")
# print(result)
pass
-218
View File
@@ -1,218 +0,0 @@
import os
import json
import asyncio
import re
from typing import List, Dict, Any, Optional
from square import AsyncSquare
import get_sq_env
class AsyncSquareClient:
"""Asynchronous client for the Square API with focus on catalog operations"""
def __init__(
self,
token: Optional[str] = None,
environment: Optional[str] = None,
logger=None,
):
"""
Initialize the AsyncSquareClient.
Args:
token (str, optional): Square API token. If not provided, fetches from environment.
environment (str, optional): Square environment (sandbox/production). If not provided, fetches from environment.
logger: Optional logger object for logging operations.
"""
self.token = token or get_sq_env.SQUARE_TOKEN
self.environment = environment or get_sq_env.environment
self.lego_cat_id = get_sq_env.LEGO_CAT_ID
self.client = AsyncSquare(token=self.token, environment=self.environment) # type: ignore
self.logger = logger
@staticmethod
def get_set_number(text: str) -> int:
"""
Extract a five-digit LEGO set number from a string.
Args:
text (str): The text to search for a set number.
Returns:
int: The extracted set number.
Raises:
ValueError: If no five-digit number is found.
"""
numbers = re.findall(r"\d{5}", text)
if not numbers:
raise ValueError(f"No five-digit number found in string {text}")
return int(numbers[0])
async def get_item(self, sku: str) -> Dict[str, str]:
"""
Asynchronously retrieves item details from Square Catalog using a SKU.
Args:
sku (str): The SKU or text filter to search for the item.
Returns:
dict: A dictionary containing the item's name and ID.
Raises:
IndexError: If no items are found matching the SKU.
Exception: If there is an error communicating with the Square API.
"""
if self.logger:
self.logger.debug(f"Searching for item with SKU: {sku}")
result = await self.client.catalog.search_items(text_filter=sku, limit=1)
result_json = json.loads(result.model_dump_json())
try:
name = result_json["items"][0]["item_data"]["name"]
itm_id = result_json["items"][0]["id"]
if self.logger:
self.logger.info(f"Found item: {name} with ID: {itm_id}")
return {"name": name, "id": itm_id}
except (KeyError, IndexError) as e:
if self.logger:
self.logger.error(f"Error retrieving item with SKU {sku}: {str(e)}")
raise
async def update_item(self, lego_set: Dict[str, Any]) -> bool:
"""
Asynchronously updates an item in the Square catalog based on provided LEGO set information.
Args:
lego_set (dict): A dictionary containing LEGO set details including:
- sets: List containing set information
- pw_name: PriceWise system name
- set_adjustredprice: Adjusted retail price
Returns:
bool: True if update was successful.
Raises:
KeyError: If required keys are missing from input dictionary.
Exception: For Square API communication errors.
"""
if self.logger:
self.logger.debug(
f"Updating item with set info: {lego_set['sets'][0]['number']}"
)
try:
set_name = lego_set["sets"][0]["name"]
set_number = lego_set["sets"][0]["number"]
set_upc = lego_set["sets"][0]["barcode"]["UPC"]
set_image = lego_set["sets"][0]["image"]["thumbnailURL"]
pw_name = lego_set["pw_name"]
set_adjustedprice = lego_set["set_adjustredprice"]
result = await self.client.catalog.search_items(
text_filter=set_number, limit=1
)
itm_id = json.loads(result.model_dump_json())["items"][0]["id"]
if self.logger:
self.logger.info(f"Successfully updated item {set_number}")
# TODO: Implement actual update logic here
# await self.client.catalog.object.upsert()
return True
except Exception as e:
if self.logger:
self.logger.error(f"Error updating item: {str(e)}")
raise
async def get_category_items(
self, cat_id: Optional[str] = None
) -> List[Dict[str, Any]]:
"""
Return a list of items within the specified category ID.
Args:
cat_id (str, optional): The category ID to search for items.
If not provided, uses the default LEGO category ID.
Returns:
list[dict]: A list of dictionaries, each containing:
- 'id': The item's ID
- 'name': The item's name
- 'number': The LEGO set number
"""
category_id = cat_id or self.lego_cat_id
if self.logger:
self.logger.debug(f"Fetching items for category: {category_id}")
try:
# Create a properly structured query for the Square API
search_request = {
"object_types": ["ITEM"],
"query": {
"exact_query": {
"attribute_name": "category_id",
"attribute_value": category_id,
}
},
}
result = await self.client.catalog.search(**search_request)
result_json = json.loads(result.model_dump_json())
items = [
{
"id": obj["id"],
"name": obj.get("item_data", {}).get("name", "Unknown"),
"number": self.get_set_number(
obj.get("item_data", {}).get("name", "")
),
}
for obj in result_json.get("objects", [])
]
if self.logger:
self.logger.info(f"Found {len(items)} items in category {category_id}")
return items
except Exception as e:
if self.logger:
self.logger.error(f"Error fetching category items: {str(e)}")
raise
# Provide acat2id as a module-level function for backward compatibility
async def acat2id(cat_id: str) -> List[Dict[str, Any]]:
"""
Backward compatibility function for getting category items.
Args:
cat_id (str): The category ID to search for items.
Returns:
list[dict]: A list of dictionaries containing item information.
"""
client = AsyncSquareClient()
return await client.get_category_items(cat_id)
async def test() -> None:
"""Test function to demonstrate class usage"""
client = AsyncSquareClient()
print(f"@[DEBUG] ENV:\t{client.environment}")
print(f"@[DEBUG] CATID:\t{client.lego_cat_id}")
result = await client.get_category_items()
print("\nOBJECTS IN LEGO CATEGORY:")
for item in result:
print(f"ID: {item['id']}, Name: {item['name']}")
if __name__ == "__main__":
asyncio.run(test())
-277
View File
@@ -1,277 +0,0 @@
"""
Clean refactored version of abricksetapi.py without debug logging.
This module provides async functions for interacting with Brickset API
to retrieve LEGO set information.
"""
import os
import aiohttp
import json
import dotenv
from typing import Dict, Any, Optional
from datetime import datetime, timedelta
from . import throttler
dotenv.load_dotenv()
class AsyncBricksetAPI:
"""Asynchronous client for the Brickset API."""
def __init__(self, logger=None):
"""
Initialize Brickset API client.
Args:
logger: Optional logger instance for structured logging
"""
self.username = os.getenv("BRICKSET_USERNAME")
self.password = os.getenv("BRICKSET_PASS")
self.api_key = os.getenv("BRICKSET_API_KEY")
self.headers = {"UserAgent": "studspider/0.1"}
self.throttler = throttler.GlobalThrottler(rate_limit=500, period=60)
self.user_hash: Optional[str] = None
self.logger = logger
# API usage tracking (1000 requests per day limit)
self.daily_requests_used: Optional[int] = None
self.daily_requests_limit = 1000
self.usage_last_checked: Optional[datetime] = None
self.usage_check_interval = timedelta(hours=1)
async def login(self) -> Dict[str, Any]:
"""
Authenticate with Brickset API.
Returns:
dict: Authentication response containing user hash
Raises:
Exception: If authentication fails
"""
url = "https://brickset.com/api/v3.asmx/login"
params = {
"username": self.username,
"password": self.password,
"apiKey": self.api_key,
}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers
) as response:
if self.logger:
self.logger.debug(f"Login response status: {response.status}")
data = await response.read()
result = json.loads(data.decode("utf-8"))
self.user_hash = result.get("hash")
if self.logger:
self.logger.info("Successfully authenticated with Brickset API")
return result
async def get_key_usage_stats(self) -> Dict[str, Any]:
"""
Fetch API key usage statistics from Brickset.
Returns:
dict: Usage stats with apiKeyUsage array containing dateStamp and count
"""
url = "https://brickset.com/api/v3.asmx/getKeyUsageStats"
params = {"apiKey": self.api_key}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers
) as response:
if response.status == 200:
data = await response.read()
return json.loads(data.decode("utf-8"))
else:
raise Exception(
f"Failed to get usage stats: HTTP {response.status}"
)
async def check_and_update_usage(self) -> bool:
"""
Check current API usage and update internal counter.
Automatically called before each request if needed.
Returns:
bool: True if under limit, False if at/over limit
"""
now = datetime.now()
# Check usage if we haven't checked yet or if it's been over an hour
should_check = (
self.daily_requests_used is None
or self.usage_last_checked is None
or (now - self.usage_last_checked) > self.usage_check_interval
)
if should_check:
try:
stats = await self.get_key_usage_stats()
usage_array = stats.get("apiKeyUsage", [])
if usage_array and len(usage_array) > 0:
today_stats = usage_array[0] # Most recent day
self.daily_requests_used = today_stats.get("count", 0)
self.usage_last_checked = now
if self.logger:
self.logger.info(
f"API usage: {self.daily_requests_used}/{self.daily_requests_limit}"
)
except Exception as e:
if self.logger:
self.logger.warning(f"Failed to check API usage: {e}")
return True # Don't block requests if usage check fails
# Check if we're over the limit
if self.daily_requests_used is not None:
if self.daily_requests_used >= self.daily_requests_limit:
if self.logger:
self.logger.error(
f"Daily API limit reached: {self.daily_requests_used}/{self.daily_requests_limit}"
)
return False
# Warn when approaching limit (90%)
if self.daily_requests_used >= self.daily_requests_limit * 0.9:
if self.logger:
self.logger.warning(
f"Approaching daily API limit: {self.daily_requests_used}/{self.daily_requests_limit}"
)
return True
async def get_set(self, set_number: int) -> Dict[str, Any]:
"""
Fetch LEGO set information from Brickset API.
Args:
set_number: LEGO set number (e.g., 10338)
Returns:
dict: Set information including:
- set_num: LEGO set number
- set_name: Official set name
- set_theme: Theme (e.g., "Icons")
- set_image: Thumbnail URL
- set_barcode: UPC or EAN barcode
- set_retailprice: US retail price
- set_adjustredprice: Retail price + 10% markup
- pw_name: Formatted name for Square (e.g., "10338 - ICONS - Bumblebee")
- requires_manual_price: True if price >= $110
Raises:
ValueError: If set not found
Exception: For API errors or if daily limit exceeded
"""
# Check API usage before making request
if not await self.check_and_update_usage():
raise Exception(
f"Brickset API daily limit reached ({self.daily_requests_limit} requests/day). "
"Please try again tomorrow."
)
if not self.user_hash:
await self.login()
url = "https://brickset.com/api/v3.asmx/getSets"
params = {
"apiKey": self.api_key,
"userHash": self.user_hash,
"params": json.dumps({"setNumber": f"{set_number}-1"}),
}
async with aiohttp.ClientSession() as session:
async with session.get(
url, params=params, headers=self.headers
) as response:
if self.logger:
self.logger.debug(f"Get set response status: {response.status}")
if response.status == 200:
data = await response.read()
result = json.loads(data.decode("utf-8"))
if not result.get("sets"):
error_msg = f"No data found for set {set_number}"
if self.logger:
self.logger.error(error_msg)
raise ValueError(error_msg)
if self.logger:
self.logger.info(
f"Successfully retrieved data for set {set_number}"
)
set_data = result["sets"][0]
# Get UPC barcode, fall back to EAN if not available
barcode_data = set_data.get("barcode", {})
upc_code = barcode_data.get("UPC") or barcode_data.get("EAN")
# Extract set information
our_set = {
"set_num": set_data["number"],
"set_name": set_data["name"],
"set_theme": set_data["theme"],
"set_image": set_data["image"]["thumbnailURL"],
"set_barcode": upc_code,
"set_retailprice": round(
set_data["LEGOCom"]["US"]["retailPrice"]
),
}
# Calculate adjusted price (10% markup)
retail_price = our_set["set_retailprice"]
our_set["set_adjustredprice"] = retail_price + (
(10 / 100) * retail_price
)
# Format name for Square inventory
our_set["pw_name"] = (
f"{our_set['set_num']} - {our_set['set_theme'].upper()} - {our_set['set_name']}"
)
# Flag for manual price review if needed
if our_set["set_retailprice"] >= 110:
our_set["requires_manual_price"] = True
return our_set
elif response.status == 500:
error_msg = (
"Internal Server Error: Please check API key and parameters."
)
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
else:
error_msg = f"Unexpected response: {response.status}"
if self.logger:
self.logger.error(error_msg)
raise Exception(error_msg)
async def throttled_get_set(self, set_number: int) -> Dict[str, Any]:
"""
Get set information with rate limiting (500 requests per 60 seconds).
Args:
set_number: LEGO set number
Returns:
dict: Set information
Raises:
ValueError: If set not found
Exception: For API errors
"""
async with self.throttler:
return await self.get_set(set_number)
-426
View File
@@ -1,426 +0,0 @@
"""
This module provides async functions for interacting with Square Catalog API
to manage LEGO inventory items.
"""
import json
import asyncio
import uuid
import re
import aiohttp
from square.core.api_error import ApiError
from square import AsyncSquare
from . import get_sq_env
def get_set_number(text: str) -> int:
"""Extract 5-digit LEGO set number from text."""
numbers = re.findall(r"\d{5}", text)
if not numbers:
raise ValueError("No five-digit number found in string")
return int(numbers[0])
# Configuration
SQUARE_TOKEN = get_sq_env.SQUARE_TOKEN
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
ENVIRONMENT = get_sq_env.environment
# Initialize Square client
client = AsyncSquare(token=SQUARE_TOKEN, environment=ENVIRONMENT)
async def get_itm(sku: str) -> dict:
"""
Retrieve item details from Square Catalog using a SKU.
Args:
sku: The SKU or text filter to search for
Returns:
dict: Item name and ID
Raises:
IndexError: If no items found
Exception: For API errors
"""
result = await client.catalog.search_items(text_filter=sku, limit=1)
result_json = json.loads(result.model_dump_json())
name = result_json["items"][0]["item_data"]["name"]
itm_id = result_json["items"][0]["id"]
return {"name": name, "id": itm_id}
async def get_item_by_id(item_id: str) -> dict:
"""
Retrieve full item details by Square catalog item ID.
Args:
item_id: Square catalog item ID
Returns:
dict: Item details including name, category_id, variations, version
Raises:
ValueError: If item not found
Exception: For API errors
"""
result = await client.catalog.batch_get(
object_ids=[item_id],
include_related_objects=False,
)
result_json = json.loads(result.model_dump_json())
if not result_json.get("objects"):
raise ValueError(f"Item {item_id} not found (may have been deleted)")
obj = result_json["objects"][0]
item_data = obj.get("item_data", {})
return {
"id": obj["id"],
"item_data": item_data,
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
}
async def get_item_by_variation_id(variation_id: str) -> dict:
"""
Retrieve parent item details from a variation ID.
Webhooks send variation IDs, but we need the parent item.
Searches all items in LEGO category to find the matching variation.
Args:
variation_id: Square catalog item variation ID
Returns:
dict: Parent item details
Raises:
ValueError: If variation/item not found
Exception: For API errors
"""
cursor = None
page_count = 0
while True:
page_count += 1
search_params = {
"object_types": ["ITEM"],
"query": {
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
"limit": 1000,
}
if cursor:
search_params["cursor"] = cursor
result = await client.catalog.search(**search_params)
result_json = json.loads(result.model_dump_json())
if not result_json.get("objects"):
if page_count == 1:
raise ValueError("No items found in LEGO category")
break
# Search items on this page for matching variation
for obj in result_json["objects"]:
item_data = obj.get("item_data", {})
variations = item_data.get("variations", [])
for variation in variations:
if variation.get("id") == variation_id:
# Found parent item
category_id = item_data.get("category_id") or LEGO_CAT_ID
return {
"id": obj["id"],
"name": item_data.get("name", ""),
"description": item_data.get("description", ""),
"category_id": category_id,
"variations": variations,
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
}
cursor = result_json.get("cursor")
if not cursor:
break
raise ValueError(
f"Variation {variation_id} not found in LEGO category (searched {page_count} pages)"
)
async def upload_item_image(item_id: str, image_url: str) -> bool:
"""
Download image from URL and upload to Square catalog item.
Args:
item_id: Square catalog item ID
image_url: URL to download image from (e.g., Brickset thumbnail)
Returns:
bool: True if successful
"""
try:
# Download image
async with aiohttp.ClientSession() as session:
async with session.get(image_url) as resp:
if resp.status != 200:
return False
image_data = await resp.read()
# Determine content type
content_type = "image/jpeg"
if image_url.lower().endswith(".png"):
content_type = "image/png"
elif image_url.lower().endswith(".gif"):
content_type = "image/gif"
# Upload to Square
result = await client.catalog.images.create(
image_file=image_data,
request={
"idempotency_key": str(uuid.uuid4()),
"image": {
"type": "IMAGE",
"id": f"#IMAGE_{item_id}",
"image_data": {"name": "LEGO Set Image"},
},
"object_id": item_id,
},
image_file_name=f"lego_{item_id}.jpg",
content_type=content_type,
)
result_json = json.loads(result.model_dump_json())
return result_json.get("image") is not None
except Exception:
return False
async def update_itm(item_id: str, lego_set: dict, item_data: dict = None) -> bool:
"""
Update Square catalog item with Brickset LEGO set data.
Updates name, description, price, SKU, UPC, and category while preserving
all other item fields (location settings, tax, etc).
Args:
item_id: Square catalog item ID to update
lego_set: Dict containing LEGO set details from Brickset:
- set_num: LEGO set number (e.g., "75192")
- set_name: Official LEGO set name
- set_theme: LEGO theme (e.g., "Star Wars")
- set_barcode: UPC barcode
- set_image: Thumbnail URL
- pw_name: Formatted name (e.g., "75192 - STAR WARS - Millennium Falcon")
- set_adjustredprice: Retail price with 10% markup
item_data: Optional pre-fetched item data with category_id
Returns:
bool: True if successful
Raises:
KeyError: If expected keys missing from lego_set
Exception: For API errors
"""
try:
# Extract LEGO set data
set_number = lego_set["set_num"]
set_name = lego_set["set_name"]
set_theme = lego_set["set_theme"]
set_upc = lego_set["set_barcode"]
set_image = lego_set["set_image"]
pw_name = lego_set["pw_name"]
set_adjustedprice = lego_set["set_adjustredprice"]
price_in_cents = int(set_adjustedprice * 100)
# Retry logic for version conflicts
max_retries = 3
retry_delay = 0.5
for attempt in range(max_retries):
try:
# Fetch fresh item data for latest version
fresh_item = await get_item_by_id(item_id)
except ValueError:
# Item was deleted
return False
variations = fresh_item.get("item_data", {}).get("variations", [])
current_version = fresh_item.get("version")
# Get variation details
if variations:
variation_id = variations[0]["id"]
variation_version = variations[0].get("version")
else:
variation_id = None
variation_version = None
# Determine category ID
# batch_get doesn't return category_id, so use from item_data or fallback
if item_data and item_data.get("category_id"):
category_id = item_data.get("category_id")
else:
category_id = LEGO_CAT_ID
# Preserve all existing item_data and update only what we need
existing_item_data = fresh_item.get("item_data", {})
existing_item_data["name"] = pw_name
existing_item_data["description"] = f"{set_name} - {set_theme}"
existing_item_data["category_id"] = category_id
# Update only the specific variation
existing_variations = existing_item_data.get("variations", [])
updated_variations = []
variation_found = False
for existing_var in existing_variations:
if existing_var.get("id") == variation_id:
# Preserve all variation fields, update only specific ones
updated_var = existing_var.copy()
updated_var_data = existing_var.get(
"item_variation_data", {}
).copy()
updated_var_data["price_money"] = {
"amount": price_in_cents,
"currency": "USD",
}
updated_var_data["sku"] = set_number
updated_var_data["upc"] = set_upc if set_upc else None
updated_var["item_variation_data"] = updated_var_data
if variation_version is not None:
updated_var["version"] = variation_version
updated_variations.append(updated_var)
variation_found = True
else:
updated_variations.append(existing_var)
# Create new variation if not found
if not variation_found:
new_variation = {
"type": "ITEM_VARIATION",
"id": variation_id if variation_id else "#TEMP_VARIATION_ID",
"item_variation_data": {
"item_id": item_id,
"name": "Regular",
"pricing_type": "FIXED_PRICING",
"price_money": {
"amount": price_in_cents,
"currency": "USD",
},
"sku": set_number,
"upc": set_upc if set_upc else None,
},
}
if variation_version is not None:
new_variation["version"] = variation_version
updated_variations.append(new_variation)
existing_item_data["variations"] = updated_variations
# Build upsert request
upsert_body = {
"idempotency_key": str(uuid.uuid4()),
"object": {
"type": "ITEM",
"id": item_id,
"version": current_version,
"item_data": existing_item_data,
},
}
# Perform upsert
try:
result = await client.catalog.object.upsert(
idempotency_key=upsert_body["idempotency_key"],
object=upsert_body["object"],
)
result_json = json.loads(result.model_dump_json())
if result_json.get("catalog_object"):
# Upload image if available (currently disabled)
# if set_image:
# await upload_item_image(item_id, set_image)
return True
else:
return False
except ApiError as e:
# Handle version conflicts with retry
if e.status_code == 400 and e.body and isinstance(e.body, dict):
errors = e.body.get("errors", [])
if any(err.get("code") == "VERSION_MISMATCH" for err in errors):
if attempt < max_retries - 1:
await asyncio.sleep(retry_delay)
retry_delay *= 2
continue
else:
return False
raise
except Exception:
return False
async def acat2id(cat_id: str) -> list[dict]:
"""
List all items in a category.
Args:
cat_id: Category ID to search
Returns:
list[dict]: List of items with id, name, set number, updated_at, version
"""
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": cat_id,
}
},
)
result_json = json.loads(result.model_dump_json())
items = []
for obj in result_json.get("objects", []):
item_name = obj.get("item_data", {}).get("name", "")
set_number = None
if re.findall(r"\d{5}", item_name):
try:
set_number = get_set_number(item_name)
except ValueError:
pass
items.append(
{
"id": obj["id"],
"name": item_name or "Unknown",
"number": set_number,
"updated_at": obj.get("updated_at"),
"version": obj.get("version"),
}
)
return items
-110
View File
@@ -1,110 +0,0 @@
"""
Convert P7B (PKCS#7) certificate to PEM format for use with the server
Use this if you have a P7B file and need to extract PEM certificates
"""
import subprocess
import sys
import os
def convert_p7b_to_pem(p7b_file, cert_output="cert.pem", extract_key=False):
"""
Convert P7B certificate to PEM format
Args:
p7b_file: Input P7B (PKCS#7) file
cert_output: Output PEM certificate file
extract_key: If True, try to extract private key (rarely works with P7B)
"""
# Check if OpenSSL is available
try:
subprocess.run(["openssl", "version"], capture_output=True, check=True)
except (subprocess.CalledProcessError, FileNotFoundError):
print("ERROR: OpenSSL not found!")
print("\nFor Windows, you can:")
print(
"1. Install OpenSSL from: https://slproweb.com/products/Win32OpenSSL.html"
)
print("2. Or use Git Bash which includes OpenSSL")
print("3. Or use WSL (Windows Subsystem for Linux)")
return False
# Check if input file exists
if not os.path.exists(p7b_file):
print(f"ERROR: P7B file not found: {p7b_file}")
print(f"Current directory: {os.getcwd()}")
return False
print(f"Converting {p7b_file} to PEM format...")
# Convert P7B to PEM
cmd = ["openssl", "pkcs7", "-print_certs", "-in", p7b_file, "-out", cert_output]
try:
result = subprocess.run(cmd, check=True, capture_output=True, text=True)
if os.path.exists(cert_output):
print(f"\n✓ PEM certificate created: {os.path.abspath(cert_output)}")
print(f" File size: {os.path.getsize(cert_output)} bytes")
# Check what was extracted
with open(cert_output, "r") as f:
content = f.read()
cert_count = content.count("BEGIN CERTIFICATE")
print(f" Certificates found: {cert_count}")
print("\n" + "=" * 60)
print("IMPORTANT:")
print("=" * 60)
print("⚠️ P7B files usually contain ONLY certificates, not private keys!")
print("\nTo run an HTTPS server, you need:")
print("1. Certificate (cert.pem) - ✓ Extracted")
print("2. Private Key (key.pem) - ✗ NOT in P7B file")
print("\nIf you don't have the private key:")
print("- Generate a new certificate pair: python generate_cert.py")
print("- Or obtain the private key from your certificate provider")
print("=" * 60)
return True
else:
print("ERROR: Output file was not created")
return False
except subprocess.CalledProcessError as e:
print(f"ERROR: Conversion failed: {e}")
if e.stderr:
print(f"OpenSSL error: {e.stderr}")
return False
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(
description="Convert P7B (PKCS#7) certificate to PEM format"
)
parser.add_argument("p7b_file", help="Input P7B certificate file")
parser.add_argument(
"--output",
default="cert.pem",
help="Output PEM certificate file (default: cert.pem)",
)
args = parser.parse_args()
print("=" * 60)
print("P7B to PEM Certificate Converter")
print("=" * 60)
print()
success = convert_p7b_to_pem(args.p7b_file, args.output)
if success:
print("\n✓ Conversion complete!")
print("\nNext steps:")
print("1. If you have the private key separately, use it with --ssl-key")
print("2. If not, generate a new certificate: python generate_cert.py")
else:
sys.exit(1)
-166
View File
@@ -1,166 +0,0 @@
"""
Convert PEM certificate to P7B (PKCS#7) format for Square webhooks
Square requires P7B format for webhook signature verification
"""
import subprocess
import sys
import os
def convert_pem_to_p7b(cert_file="cert.pem", output_file="cert.p7b"):
"""
Convert PEM certificate to P7B (PKCS#7) format
Args:
cert_file: Input PEM certificate file
output_file: Output P7B file
"""
# Check if OpenSSL is available
try:
subprocess.run(["openssl", "version"], capture_output=True, check=True)
except (subprocess.CalledProcessError, FileNotFoundError):
print("ERROR: OpenSSL not found!")
print("\nFor Windows, you can:")
print(
"1. Install OpenSSL from: https://slproweb.com/products/Win32OpenSSL.html"
)
print("2. Or use Git Bash which includes OpenSSL")
print("3. Or use WSL (Windows Subsystem for Linux)")
return False
# Check if input file exists
if not os.path.exists(cert_file):
print(f"ERROR: Certificate file not found: {cert_file}")
print(f"Current directory: {os.getcwd()}")
print("\nMake sure you've generated the certificate first:")
print(" python generate_cert.py")
return False
print(f"Converting {cert_file} to P7B format...")
# Convert PEM to P7B (PKCS#7)
# -certfile can include intermediate certificates if needed
cmd = [
"openssl",
"crl2pkcs7",
"-nocrl",
"-certfile",
cert_file,
"-out",
output_file,
]
try:
result = subprocess.run(cmd, check=True, capture_output=True, text=True)
if os.path.exists(output_file):
print(f"\n✓ P7B certificate created: {os.path.abspath(output_file)}")
print(f" File size: {os.path.getsize(output_file)} bytes")
print("\n" + "=" * 60)
print("Square Webhook Configuration:")
print("=" * 60)
print(f"1. Upload this file to Square Developer Dashboard:")
print(f" {os.path.abspath(output_file)}")
print("\n2. In Square Dashboard:")
print(" - Go to Webhooks settings")
print(" - Click 'Manage Notification Settings'")
print(" - Upload the P7B file for signature verification")
print("\n3. Square will use this to verify webhook signatures")
print("=" * 60)
# Show file contents preview
print("\nP7B file preview (first 500 chars):")
with open(output_file, "r") as f:
content = f.read(500)
print(content)
if len(content) == 500:
print("...")
return True
else:
print("ERROR: Output file was not created")
return False
except subprocess.CalledProcessError as e:
print(f"ERROR: Conversion failed: {e}")
if e.stderr:
print(f"OpenSSL error: {e.stderr}")
return False
def convert_with_chain(cert_file="cert.pem", chain_file=None, output_file="cert.p7b"):
"""
Convert PEM certificate with certificate chain to P7B format
Use this if you have intermediate certificates
Args:
cert_file: Your certificate file
chain_file: Intermediate/chain certificate file (optional)
output_file: Output P7B file
"""
if not chain_file:
return convert_pem_to_p7b(cert_file, output_file)
# Check files exist
for f in [cert_file, chain_file]:
if not os.path.exists(f):
print(f"ERROR: File not found: {f}")
return False
print(f"Converting {cert_file} with chain {chain_file} to P7B format...")
# Create combined cert file
combined = "combined_cert.pem"
with open(combined, "w") as out:
with open(cert_file, "r") as f:
out.write(f.read())
with open(chain_file, "r") as f:
out.write(f.read())
result = convert_pem_to_p7b(combined, output_file)
# Clean up combined file
if os.path.exists(combined):
os.remove(combined)
return result
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(
description="Convert PEM certificate to P7B (PKCS#7) for Square webhooks"
)
parser.add_argument(
"--cert",
default="cert.pem",
help="Input PEM certificate file (default: cert.pem)",
)
parser.add_argument(
"--chain", default=None, help="Optional: Intermediate certificate chain file"
)
parser.add_argument(
"--output", default="cert.p7b", help="Output P7B file (default: cert.p7b)"
)
args = parser.parse_args()
print("=" * 60)
print("PEM to P7B Certificate Converter for Square Webhooks")
print("=" * 60)
print()
if args.chain:
success = convert_with_chain(args.cert, args.chain, args.output)
else:
success = convert_pem_to_p7b(args.cert, args.output)
if not success:
sys.exit(1)
print("\n✓ Conversion complete!")
+7
View File
@@ -0,0 +1,7 @@
CREATE TABLE "REVIEW" (
"sq_object" TEXT NOT NULL UNIQUE,
"time_added" INTEGER,
"set_number" INTEGER,
"time_modified" INTEGER,
PRIMARY KEY("sq_object")
)
-76
View File
@@ -1,76 +0,0 @@
"""
Generate self-signed SSL certificate for testing HTTPS webhooks
Run this script to create cert.pem and key.pem files
"""
import subprocess
import sys
import os
def generate_certificate():
"""Generate self-signed certificate using OpenSSL"""
cert_file = "cert.pem"
key_file = "key.pem"
# Check if OpenSSL is available
try:
subprocess.run(["openssl", "version"], capture_output=True, check=True)
except (subprocess.CalledProcessError, FileNotFoundError):
print("ERROR: OpenSSL not found!")
print("\nFor Windows, you can:")
print(
"1. Install OpenSSL from: https://slproweb.com/products/Win32OpenSSL.html"
)
print("2. Or use Git Bash which includes OpenSSL")
print("3. Or use WSL (Windows Subsystem for Linux)")
return False
print("Generating self-signed certificate...")
print("This will create cert.pem and key.pem files")
print()
# Generate certificate valid for 365 days
cmd = [
"openssl",
"req",
"-x509",
"-newkey",
"rsa:4096",
"-keyout",
key_file,
"-out",
cert_file,
"-days",
"365",
"-nodes", # No passphrase
"-subj",
"/C=US/ST=State/L=City/O=Organization/CN=localhost",
]
try:
subprocess.run(cmd, check=True)
print(f"\n✓ Certificate created: {os.path.abspath(cert_file)}")
print(f"✓ Private key created: {os.path.abspath(key_file)}")
print("\nYou can now run the server with HTTPS:")
print(
f" python src/test_server.py --ssl-cert {cert_file} --ssl-key {key_file} --hook-port 8443"
)
print("\nOr webhook-only HTTPS:")
print(
f" python src/test_server.py --ssl-cert {cert_file} --ssl-key {key_file} --hook-port 8443 --hook-ssl-only"
)
print(
"\nNote: Self-signed certificates will show security warnings in browsers."
)
print("For production, use certificates from a trusted CA like Let's Encrypt.")
return True
except subprocess.CalledProcessError as e:
print(f"ERROR: Failed to generate certificate: {e}")
return False
if __name__ == "__main__":
if not generate_certificate():
sys.exit(1)
-89
View File
@@ -1,89 +0,0 @@
"""
List all Square categories with their IDs to help identify the correct LEGO category ID
"""
import asyncio
import json
from .. import asquare
from .. import get_sq_env
async def list_categories():
"""List all categories in Square catalog"""
try:
print("\n" + "=" * 60)
print("SQUARE CATALOG CATEGORIES")
print("=" * 60 + "\n")
cursor = None
page_count = 0
total_categories = 0
while True:
page_count += 1
print(f"Fetching page {page_count}...")
search_params = {
"object_types": ["CATEGORY"],
"limit": 1000,
}
if cursor:
search_params["cursor"] = cursor
result = await asquare.client.catalog.search(**search_params)
result_json = json.loads(result.model_dump_json())
if not result_json.get("objects"):
if page_count == 1:
print("No categories found!")
break
for obj in result_json["objects"]:
category_data = obj.get("category_data", {})
category_id = obj.get("id")
category_name = category_data.get("name", "Unnamed")
print(f"\nCategory: {category_name}")
print(f" ID: {category_id}")
# Count items in this category
items_result = await asquare.client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": category_id,
}
},
limit=1,
)
items_json = json.loads(items_result.model_dump_json())
item_count = len(items_json.get("objects", []))
print(f" Items: {item_count}+ items")
total_categories += 1
cursor = result_json.get("cursor")
if not cursor:
break
print("\n" + "=" * 60)
print(f"Total categories found: {total_categories}")
print("=" * 60 + "\n")
# Show current configured category
print(f"Currently configured LEGO_CAT_ID: {get_sq_env.LEGO_CAT_ID}")
print("\nUpdate .env file with the correct category ID if needed:")
print('SQUARE_LEGO_CAT_ID = "YOUR_CATEGORY_ID_HERE"')
print()
except Exception as e:
print(f"Error listing categories: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
asyncio.run(list_categories())
-162
View File
@@ -1,162 +0,0 @@
"""
Migrate SKU values to GTIN/UPC field for all LEGO items.
This script reads existing Square catalog items in the LEGO category,
and for each variation that has a SKU but no UPC, it moves the SKU
value to the UPC field (leaving SKU empty or optionally extracting
set number from item name).
"""
import asyncio
import json
import re
import sys
import uuid
from pathlib import Path
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))
import get_sq_env
from square import AsyncSquare
SQUARE_LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
async def migrate_sku_to_gtin():
"""Migrate SKU values to GTIN for all LEGO items."""
client = AsyncSquare(
token=get_sq_env.SQUARE_TOKEN,
environment=get_sq_env.environment,
)
print(f"[@INFO] Searching for items in LEGO category {SQUARE_LEGO_CAT_ID}...")
# Search all items in LEGO category
cursor = None
page = 1
total_items = 0
updated_items = 0
while True:
print(f"[@INFO] Processing page {page}...")
search_params = {
"object_types": ["ITEM"],
"query": {
"exact_query": {
"attribute_name": "category_id",
"attribute_value": SQUARE_LEGO_CAT_ID,
}
},
"limit": 100,
}
if cursor:
search_params["cursor"] = cursor
result = await client.catalog.search(**search_params)
result_json = json.loads(result.model_dump_json())
items = result_json.get("objects", [])
cursor = result_json.get("cursor")
if not items:
break
# Process each item
for item in items:
item_id = item["id"]
item_data = item.get("item_data", {})
item_name = item_data.get("name", "")
variations = item_data.get("variations", [])
item_version = item.get("version")
total_items += 1
needs_update = False
updated_variations = []
# Extract set number from item name if it matches LEGO format
set_number = None
match = re.search(r"^(\d{5})\s*-", item_name)
if match:
set_number = match.group(1)
# Check each variation
for var in variations:
var_data = var.get("item_variation_data", {})
var_id = var.get("id")
var_version = var.get("version")
sku = var_data.get("sku")
upc = var_data.get("upc")
# If SKU exists but UPC doesn't, migrate
if sku and not upc:
print(f"[@INFO] Item {item_id} ({item_name})")
print(f"[@INFO] Variation {var_id}: Moving SKU '{sku}' to UPC")
# Preserve all existing variation data and only modify what we need
updated_var = var.copy() # Shallow copy the entire variation
updated_var_data = var_data.copy() # Copy the variation_data
# Update only UPC and SKU fields
updated_var_data["upc"] = sku # Move SKU value to UPC
updated_var_data["sku"] = (
set_number if set_number else None
) # Set to LEGO set number
updated_var["item_variation_data"] = updated_var_data
updated_variations.append(updated_var)
needs_update = True
else:
# Keep variation as-is
updated_variations.append(var)
# Update item if needed
if needs_update:
try:
# Get fresh item data to ensure we have latest version
fresh_result = await client.catalog.batch_get(
object_ids=[item_id],
include_related_objects=False,
)
fresh_json = json.loads(fresh_result.model_dump_json())
fresh_item = fresh_json["objects"][0]
fresh_item_data = fresh_item.get("item_data", {})
fresh_version = fresh_item.get("version")
# Update variations while preserving all other fields
fresh_item_data["variations"] = updated_variations
await client.catalog.object.upsert(
idempotency_key=str(uuid.uuid4()),
object={
"type": "ITEM",
"id": item_id,
"version": fresh_version,
"item_data": fresh_item_data,
},
)
print(f"[@INFO] ✓ Successfully updated item {item_id}")
updated_items += 1
# Small delay to avoid rate limits
await asyncio.sleep(0.1)
except Exception as e:
print(f"[@ERROR] Failed to update item {item_id}: {e}")
page += 1
# If no cursor, we're done
if not cursor:
break
print(f"\n[@INFO] Migration complete!")
print(f"[@INFO] Total items processed: {total_items}")
print(f"[@INFO] Items updated: {updated_items}")
if __name__ == "__main__":
print("[@INFO] Starting SKU to GTIN migration...\n")
asyncio.run(migrate_sku_to_gtin())
-15
View File
@@ -1,15 +0,0 @@
CREATE TABLE "REVIEW" (
"sq_object" TEXT NOT NULL UNIQUE,
"price" INTEGER NOT NULL,
"date_modified" INTEGER,
PRIMARY KEY("sq_object")
)
CREATE TABLE "MISSING" (
"sq_object" TEXT NOT NULL UNIQUE,
"time_added" INTEGER,
"set_number" INTEGER,
"time_modified" INTEGER,
"set_name" TEXT,
PRIMARY KEY("sq_object")
)
+113 -113
View File
@@ -1,113 +1,113 @@
import asyncio
import aiohttp.web
import uuid
import os
from base64 import b64decode
import dotenv
dotenv.load_dotenv()
SQUARE_LEGO_CAT_ID = os.environ.get("SQUARE_LEGO_CAT_ID")
from asyncio_throttle.throttler import Throttler
import asquare as sq
import abricksetapi as alego
def valid_uuid(val):
try:
uuid.UUID(str(val))
return True
except ValueError:
return False
async def fullsync():
"""'
full_db = await sq.acat2id(SQUARE_LEGO_CAT_ID)
for l_set in full_db:
l_setinfo = await alego.throttled_get_set(l_set)
await sq.update_itm(l_set, l_setinfo)"""
async def stop():
pass
async def priceonly():
pass
async def handle_post(request):
valid_cmd = ["RUNFULLSYNC", "STOP", "RUNPRICEONLY"]
async def handle_cmd(cmd: str):
if cmd not in valid_cmd:
return False
else:
return True
idp_key = request.headers.get("Idempotency-Key")
user_agent = request.headers.get("User-Agent")
if not idp_key and not valid_uuid(idp_key) and user_agent != "Kopling/0.1":
# 412 Precondition Failed
# The server does not meet one of the preconditions that the requester put on the request header fields.
return aiohttp.web.Response(status=412, text="412 Headers rejected!")
else:
data = await request.post()
reqdata = data.get(b64decode("CMD"))
if await handle_cmd(reqdata):
match reqdata:
case "RUNFULLSYNC":
print("Running full sync...")
await fullsync()
return True
case "STOP":
await stop()
return True
case "RUNPRICEONLY":
await priceonly()
return True
else:
return aiohttp.web.Response(status=400, text="400 Bad Request.")
result = {"message": f"studspider/0.1 received command: {reqdata}"}
return aiohttp.web.json_response(status=200, data=result)
app = aiohttp.web.Application()
app.router.add_post("/", handle_post)
import subprocess
def generate_toilet_banner(text, font="smblock", filter_effect=None, colors=None):
command = ["toilet", "-f", font]
if filter_effect:
command.extend(["-F", filter_effect])
if colors:
# Toilet uses a specific format for color arguments, e.g., --gay or --metal
# For general colors, you might need to use specific filters or direct color codes.
# This example focuses on filters that inherently use colors.
pass # More complex color handling would be needed for arbitrary colors
command.append(text)
try:
result = subprocess.run(command, capture_output=True, text=True, check=True)
return result.stdout
except subprocess.CalledProcessError as e:
return f"Error executing toilet: {e}\n{e.stderr}"
async def main():
throttler = Throttler(rate_limit=500, period=60) # 500 requests per minute
if __name__ == "__main__":
print(generate_toilet_banner("studspider v0.1"))
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
aiohttp.web.run_app(app, port=7432)
import asyncio
import aiohttp.web
import uuid
import os
from base64 import b64decode
import dotenv
dotenv.load_dotenv()
SQUARE_LEGO_CAT_ID = os.environ.get("SQUARE_LEGO_CAT_ID")
from asyncio_throttle.throttler import Throttler
import asquare as sq
import abricksetapi as alego
def valid_uuid(val):
try:
uuid.UUID(str(val))
return True
except ValueError:
return False
async def fullsync():
"""'
full_db = await sq.acat2id(SQUARE_LEGO_CAT_ID)
for l_set in full_db:
l_setinfo = await alego.throttled_get_set(l_set)
await sq.update_itm(l_set, l_setinfo)"""
async def stop():
pass
async def priceonly():
pass
async def handle_post(request):
valid_cmd = ["RUNFULLSYNC", "STOP", "RUNPRICEONLY"]
async def handle_cmd(cmd: str):
if cmd not in valid_cmd:
return False
else:
return True
idp_key = request.headers.get("Idempotency-Key")
user_agent = request.headers.get("User-Agent")
if not idp_key and not valid_uuid(idp_key) and user_agent != "Kopling/0.1":
# 412 Precondition Failed
# The server does not meet one of the preconditions that the requester put on the request header fields.
return aiohttp.web.Response(status=412, text="412 Headers rejected!")
else:
data = await request.post()
reqdata = data.get(b64decode("CMD"))
if await handle_cmd(reqdata):
match reqdata:
case "RUNFULLSYNC":
print("Running full sync...")
await fullsync()
return True
case "STOP":
await stop()
return True
case "RUNPRICEONLY":
await priceonly()
return True
else:
return aiohttp.web.Response(status=400, text="400 Bad Request.")
result = {"message": f"Edderkop/0.1 received command: {reqdata}"}
return aiohttp.web.json_response(status=200, data=result)
app = aiohttp.web.Application()
app.router.add_post("/", handle_post)
import subprocess
def generate_toilet_banner(text, font="smblock", filter_effect=None, colors=None):
command = ["toilet", "-f", font]
if filter_effect:
command.extend(["-F", filter_effect])
if colors:
# Toilet uses a specific format for color arguments, e.g., --gay or --metal
# For general colors, you might need to use specific filters or direct color codes.
# This example focuses on filters that inherently use colors.
pass # More complex color handling would be needed for arbitrary colors
command.append(text)
try:
result = subprocess.run(command, capture_output=True, text=True, check=True)
return result.stdout
except subprocess.CalledProcessError as e:
return f"Error executing toilet: {e}\n{e.stderr}"
async def main():
throttler = Throttler(rate_limit=500, period=60) # 500 requests per minute
if __name__ == "__main__":
print(generate_toilet_banner("Edderkop v0.1"))
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
aiohttp.web.run_app(app, port=7432)
-359
View File
@@ -1,359 +0,0 @@
"""
Statistics gathering functions for daily_stats table.
Queries Square catalog to collect metrics about LEGO inventory quality.
"""
import json
import git
from typing import Dict, Any
from square import AsyncSquare
from . import get_sq_env
from . import abricksetapi
def get_git_commit() -> str:
"""Get first 6 characters of current git commit hash"""
try:
repo = git.Repo(search_parent_directories=True)
return repo.head.commit.hexsha[:6]
except Exception:
return "unknown"
# Initialize Square client
client = AsyncSquare(token=get_sq_env.SQUARE_TOKEN, environment=get_sq_env.environment)
LEGO_CAT_ID = get_sq_env.LEGO_CAT_ID
async def get_daily_stats() -> Dict[str, Any]:
"""
Gather all metrics for daily_stats table in a single pass.
Returns:
dict: {
"total_items": int,
"enriched_items": int,
"missing_upcs": int,
"missing_images": int,
"total_catalog_value": float
}
"""
# Get all items in LEGO category
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
)
result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", [])
# Initialize counters
total_items = 0
enriched_items = 0
missing_upcs = 0
missing_images = 0
total_catalog_value = 0.0
for obj in objects:
total_items += 1
item_data = obj.get("item_data", {})
# Check if item has all enrichment data
has_upc = False
has_image = False
has_proper_name = False
item_price = 0.0
# Check variations for UPC and price
variations = item_data.get("variations", [])
if variations:
var_data = variations[0].get("item_variation_data", {})
# Check UPC
if var_data.get("upc"):
has_upc = True
else:
missing_upcs += 1
# Get price (stored in cents)
price_money = var_data.get("price_money", {})
if price_money and price_money.get("amount"):
item_price = price_money["amount"] / 100.0 # Convert cents to dollars
total_catalog_value += item_price
else:
missing_upcs += 1
# Check for image
if item_data.get("image_ids"):
has_image = True
else:
missing_images += 1
# Check if name follows format: "12345 - THEME Set Name"
name = item_data.get("name", "")
if " - " in name and name.split()[0].isdigit() and len(name.split()[0]) == 5:
has_proper_name = True
# Item is "enriched" if it has UPC, image, and proper name format
if has_upc and has_image and has_proper_name:
enriched_items += 1
return {
"total_items": total_items,
"enriched_items": enriched_items,
"missing_upcs": missing_upcs,
"missing_images": missing_images,
"total_catalog_value": round(total_catalog_value, 2),
}
async def get_brickset_cache_stats() -> Dict[str, Any]:
"""
Get Brickset API cache statistics.
Returns:
dict: {
"cache_enabled": bool,
"cache_hits": int,
"cache_misses": int,
"cache_size": int,
"cache_maxsize": int,
"hit_rate": float (0-100%)
}
"""
# Create a temporary client instance to access cache info
temp_client = abricksetapi.AsyncBricksetAPI()
# Check if get_set has cache_info (it should with @alru_cache)
if hasattr(temp_client.get_set, "cache_info"):
info = await temp_client.get_set.cache_info()
hits = info.hits
misses = info.misses
total = hits + misses
hit_rate = (hits / total * 100) if total > 0 else 0.0
return {
"cache_enabled": True,
"cache_hits": hits,
"cache_misses": misses,
"cache_size": info.currsize,
"cache_maxsize": info.maxsize,
"hit_rate": round(hit_rate, 2),
}
else:
return {
"cache_enabled": False,
"cache_hits": 0,
"cache_misses": 0,
"cache_size": 0,
"cache_maxsize": 0,
"hit_rate": 0.0,
}
async def get_daily_stats() -> Dict[str, Any]:
"""
Gather all metrics for daily_stats table in a single pass.
Returns:
dict: {
"total_items": int,
"enriched_items": int,
"missing_upcs": int,
"missing_images": int,
"total_catalog_value": float
}
"""
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
)
result_json = json.loads(result.model_dump_json())
return len(result_json.get("objects", []))
async def get_enriched_count() -> int:
"""
Count items with complete enrichment (UPC + image + proper name format).
Returns:
int: Number of fully enriched items
"""
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
)
result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", [])
enriched_count = 0
for obj in objects:
item_data = obj.get("item_data", {})
# Check UPC
has_upc = False
variations = item_data.get("variations", [])
if variations:
var_data = variations[0].get("item_variation_data", {})
has_upc = bool(var_data.get("upc"))
# Check image
has_image = bool(item_data.get("image_ids"))
# Check name format: "12345 - THEME Set Name"
name = item_data.get("name", "")
has_proper_name = False
if " - " in name:
first_part = name.split()[0]
if first_part.isdigit() and len(first_part) == 5:
has_proper_name = True
if has_upc and has_image and has_proper_name:
enriched_count += 1
return enriched_count
async def get_missing_upcs() -> int:
"""
Count items without UPC barcodes.
Returns:
int: Number of items missing UPCs
"""
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
)
result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", [])
missing_count = 0
for obj in objects:
item_data = obj.get("item_data", {})
variations = item_data.get("variations", [])
if not variations:
missing_count += 1
continue
var_data = variations[0].get("item_variation_data", {})
if not var_data.get("upc"):
missing_count += 1
return missing_count
async def get_missing_images() -> int:
"""
Count items without images.
Returns:
int: Number of items missing images
"""
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
)
result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", [])
missing_count = 0
for obj in objects:
item_data = obj.get("item_data", {})
if not item_data.get("image_ids"):
missing_count += 1
return missing_count
async def get_total_catalog_value() -> float:
"""
Calculate total dollar value of all items in LEGO category.
Returns:
float: Total catalog value in dollars
"""
result = await client.catalog.search(
object_types=["ITEM"],
query={
"exact_query": {
"attribute_name": "category_id",
"attribute_value": LEGO_CAT_ID,
}
},
)
result_json = json.loads(result.model_dump_json())
objects = result_json.get("objects", [])
total_value = 0.0
for obj in objects:
item_data = obj.get("item_data", {})
variations = item_data.get("variations", [])
if variations:
var_data = variations[0].get("item_variation_data", {})
price_money = var_data.get("price_money", {})
if price_money and price_money.get("amount"):
# Price is stored in cents, convert to dollars
total_value += price_money["amount"] / 100.0
return round(total_value, 2)
# Example usage
if __name__ == "__main__":
import asyncio
async def main():
print("Gathering daily statistics for LEGO catalog...")
stats = await get_daily_stats()
print(f"\n=== Daily Stats ===")
print(f"Total Items: {stats['total_items']}")
print(f"Enriched Items: {stats['enriched_items']}")
print(f"Missing UPCs: {stats['missing_upcs']}")
print(f"Missing Images: {stats['missing_images']}")
print(f"Total Catalog Value: ${stats['total_catalog_value']:,.2f}")
print(
f"Completion Rate: {(stats['enriched_items'] / stats['total_items'] * 100):.1f}%"
)
asyncio.run(main())
+6 -6
View File
@@ -1,6 +1,6 @@
import src.libsquare as sq
import src.bricksetapi as lego
oursq = sq.get_itm("10291")
ourlego = lego.get_set(10291)
print(f"{oursq}\n\n{ourlego}\n\n")
import src.libsquare as sq
import src.bricksetapi as lego
oursq = sq.get_itm("71417")
ourlego = lego.get_set(71417)
print(f"{oursq}\n\n{ourlego}\n\n")
+39 -39
View File
@@ -1,39 +1,39 @@
import asyncio
import aiohttp
from uuid import uuid4
from base64 import b64encode
import os
async def idp_request(cmd: str):
headers = {
"Idempotency-Key": f"{uuid4()}",
"Content-Type": "application/json",
"User-Agent": "Kopling/0.12",
}
print(f"[@DEBUG] Idempotency-Key: {headers['Idempotency-Key']}")
enc_cmd = b64encode(f"{cmd}".encode("utf-8")).decode("utf-8")
print("[@DEBUG] Encoded command:", enc_cmd)
print("[@DEBUG] Sending request..")
try:
async with aiohttp.ClientSession() as session:
async with session.post(
"http://localhost:7432", # Added http:// protocol
headers=headers,
json={"CMD": enc_cmd}, # Changed to json instead of data
) as resp:
print(f"[@DEBUG] Response status: {resp.status}")
print(f"[@DEBUG] Response body: {await resp.text()}")
except Exception as e:
print(f"\nRequest failed: {e}")
os._exit(-1)
async def main():
# Await the task instead of just creating it
await idp_request("RUNUPDATE")
if __name__ == "__main__":
asyncio.run(main())
import asyncio
import aiohttp
from uuid import uuid4
from base64 import b64encode
import os
async def idp_request(cmd: str):
headers = {
"Idempotency-Key": f"{uuid4()}",
"Content-Type": "application/json",
"User-Agent": "Kopling/0.12",
}
print(f"[@DEBUG] Idempotency-Key: {headers['Idempotency-Key']}")
enc_cmd = b64encode(f"{cmd}".encode("utf-8")).decode("utf-8")
print("[@DEBUG] Encoded command:", enc_cmd)
print("[@DEBUG] Sending request..")
try:
async with aiohttp.ClientSession() as session:
async with session.post(
"http://localhost:7432", # Added http:// protocol
headers=headers,
json={"CMD": enc_cmd}, # Changed to json instead of data
) as resp:
print(f"[@DEBUG] Response status: {resp.status}")
print(f"[@DEBUG] Response body: {await resp.text()}")
except Exception as e:
print(f"\nRequest failed: {e}")
os._exit(-1)
async def main():
# Await the task instead of just creating it
await idp_request("RUNUPDATE")
if __name__ == "__main__":
asyncio.run(main())
+128
View File
@@ -0,0 +1,128 @@
import asyncio
import json
from aiohttp import web
from base64 import b64decode
from typing import Dict, Any
import uuid
from time import time
import dotenv
import os
import re
import logging
from logging.handlers import RotatingFileHandler
from pathlib import Path
Path("./logs/").mkdir(parents=True, exist_ok=True)
import banner
import get_sq_env
import asquare as sq
from abricksetapi import AsyncBricksetAPI
dotenv.load_dotenv()
SQUARE_LEGO_CAT_ID = os.environ.get("SQUARE_LEGO_CAT_ID")
from asyncio_throttle.throttler import Throttler
# set up logger
logger = logging.getLogger("edderkop")
logger.setLevel(logging.DEBUG)
handler = RotatingFileHandler("./logs/", maxBytes=1024, backupCount=5)
logger.addHandler(handler)
lego = AsyncBricksetAPI(debug=True, logger=logger)
# logging.basicConfig(
# filename="./logs/edderkop.log", encoding="utf-8", level=logging.DEBUG
# )
UA = re.compile(r"^kopling\/?", re.IGNORECASE)
async def run_update():
set_list = await sq.acat2id(SQUARE_LEGO_CAT_ID) # type: ignore
for l_set in set_list:
print(f"[@DEBUG]{l_set}")
l_setinfo = await lego.throttled_get_set(l_set["number"])
print(f"[@DEBUG]{l_setinfo}")
# await sq.update_itm(l_set, l_setinfo)
def valid_uuid(val):
try:
uuid.UUID(str(val))
return True
except ValueError:
return False
async def request_handler(request: web.Request) -> web.Response:
"""Handle incoming POST requests with command processing"""
try:
# Verify request method
if request.method != "POST":
logger.error(
f"{time()}\tInvalid request method from %s at ", request.remote
)
return web.Response(status=405, text="Method not allowed. Use POST.")
# Get JSON data from request
try:
data = await request.json()
except json.JSONDecodeError:
logger.error("Invalid JSON from %s", request.remote)
return web.Response(
status=400, text="{time()}\tInvalid JSON in request body"
)
# Validate Idempotency-Key header
if valid_uuid(request.headers.get("Idempotency-Key")) is False:
logger.error("{time()}\tInvalid Idempotency-Key from %s", request.remote)
return web.Response(status=412, text="412 Headers rejected!")
# Validate User-Agent header
if not re.match(UA, str(request.headers.get("User-Agent"))):
logger.error("{time()}\tInvalid User-Agent from %s", request.remote)
return web.Response(status=412, text="412 Headers rejected!")
# Validate command exists
if "CMD" not in data:
logger.error("{time()}\tMissing CMD field from %s", request.remote)
return web.Response(status=400, text="Missing CMD field in request")
# Decode and process command
try:
decoded_cmd = b64decode(data["CMD"]).decode("utf-8")
print(
f"""At: {time()}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
)
logging.info(
f"{time()}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
)
# Process the command
if decoded_cmd == "RUNUPDATE":
# Add your command handling logic here
await run_update()
response_data = {"status": "success", "message": "Update started"}
logging.info(f"{time()}\tUpdate command processed successfully")
return web.json_response(response_data)
else:
return web.Response(status=400, text=f"Unknown command: {decoded_cmd}")
except Exception as e:
print(f"Error processing command: {str(e)}")
return web.Response(status=500, text=f"Server error: {str(e)}")
except Exception as e:
print(f"Request handler error: {str(e)}")
return web.Response(status=500, text="Internal server error")
app = web.Application()
app.router.add_post("/", request_handler)
if __name__ == "__main__":
print(banner.generate("Edderkop v0.1"))
print("Built for PLAY-WELL TEKNOLOGIES by contact@traps.moe\n\n")
web.run_app(app, host="localhost", port=7432)
-37
View File
@@ -1,37 +0,0 @@
import asyncio
from asyncio_throttle import Throttler as AsyncThrottler
class GlobalThrottler:
_instance = None
_lock = asyncio.Lock()
def __init__(self, rate_limit: int = 500, period: int = 60) -> None:
if not hasattr(self, "_initialized"):
self.rate_limit = rate_limit
self.period = period
self.throttler = AsyncThrottler(
rate_limit=self.rate_limit, period=self.period
)
self._initialized = True
@classmethod
async def get_instance(
cls, rate_limit: int = 500, period: int = 60
) -> "GlobalThrottler":
async with cls._lock:
if not cls._instance:
cls._instance = cls(rate_limit, period)
return cls._instance
async def acquire(self):
await self.throttler.acquire()
async def __aenter__(self):
await self.acquire()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
# asyncio_throttle.Throttler doesn't have a release() method
# It automatically manages rate limiting via acquire()
pass
@@ -1,15 +1,15 @@
import aiohttp
import asyncio
from asyncio_throttle.throttler import Throttler
throttler = Throttler(rate_limit=500, period=60) # 500 requests per minute
async def worker(throttler, session):
while True:
async with throttler:
async with session.get("http://example.com") as resp:
do_some_job_with(await resp.text())
await asyncio.sleep(0.05)
import aiohttp
import asyncio
from asyncio_throttle.throttler import Throttler
throttler = Throttler(rate_limit=500, period=60) # 500 requests per minute
async def worker(throttler, session):
while True:
async with throttler:
async with session.get("http://example.com") as resp:
do_some_job_with(await resp.text())
await asyncio.sleep(0.05)
-372
View File
@@ -1,372 +0,0 @@
import os
import json
import asyncio
from typing import Dict, Any
from fastapi import FastAPI, Request, Response, status
from fastapi.responses import JSONResponse
import src.get_sq_env as sq_env
import src.stats as stats
# Square webhook validation
from square.utils.webhooks_helper import verify_signature
# Configuration
SQUARE_WEBHOOK_SIGNATURE_KEY = sq_env.SQUARE_WEBHOOK_SIGNATURE_KEY
SQUARE_WEBHOOK_URL = sq_env.SQUARE_WEBHOOK_URL
SQUARE_LEGO_CAT_ID = sq_env.LEGO_CAT_ID
# In-memory store for processed event IDs (idempotency)
processed_event_ids = set()
MAX_STORED_EVENT_IDS = 10000
# Create FastAPI app
app = FastAPI(title="Studspider")
@app.get("/")
async def root():
"""Landing page"""
return {
"service": "Studspider",
"status": "running",
"endpoints": {
"webhook": "/hook (POST)",
"health": "/health (GET)",
"stats": "/stats (GET)",
},
}
@app.get("/health")
async def health_check():
"""Health check endpoint for monitoring"""
return {
"status": "healthy",
"signature_verification": (
"enabled" if SQUARE_WEBHOOK_SIGNATURE_KEY else "disabled"
),
}
@app.get("/stats")
async def get_stats():
"""
Get daily statistics for LEGO inventory.
Returns metrics about enrichment progress, missing data, catalog value, and cache stats.
"""
try:
daily_stats = await stats.get_daily_stats()
cache_stats = await stats.get_brickset_cache_stats()
# Calculate completion percentage
if daily_stats["total_items"] > 0:
completion_rate = (
daily_stats["enriched_items"] / daily_stats["total_items"] * 100
)
else:
completion_rate = 0.0
return {
"status": "success",
"version": "0.3.0",
"git_commit": stats.get_git_commit(),
"stats": daily_stats,
"completion_rate": round(completion_rate, 1),
"brickset_cache": cache_stats,
}
except Exception as e:
return JSONResponse(
status_code=500,
content={"status": "error", "message": str(e)},
)
@app.post("/hook")
async def webhook_handler(request: Request):
"""
Handle Square webhook events - MUST respond within 2 seconds!
Square sends POST requests with:
- x-square-hmacsha256-signature header
- JSON body with event data
- event_id for idempotency
"""
try:
# Log incoming request
print(f"\n[WEBHOOK] Incoming request from {request.client.host}")
print(f"[WEBHOOK] Method: {request.method}")
print(f"[WEBHOOK] URL: {request.url}")
print(f"[WEBHOOK] Headers: {dict(request.headers)}")
# Read raw body for signature verification
body = await request.body()
body_str = body.decode("utf-8")
print(f"[WEBHOOK] Body length: {len(body_str)} bytes")
# Get signature header
signature_header = request.headers.get("x-square-hmacsha256-signature", "")
# Verify Square webhook signature (if configured)
if SQUARE_WEBHOOK_SIGNATURE_KEY and SQUARE_WEBHOOK_URL:
if not signature_header:
print(
f"[ERROR] Missing x-square-hmacsha256-signature header from {request.client.host}"
)
return JSONResponse(
status_code=status.HTTP_401_UNAUTHORIZED,
content={"error": "Missing signature header"},
)
# Debug signature verification inputs
print(f"[DEBUG] Signature header: {signature_header[:20]}...")
print(
f"[DEBUG] Signature key (first 10 chars): {SQUARE_WEBHOOK_SIGNATURE_KEY[:10]}..."
)
print(f"[DEBUG] Notification URL: {SQUARE_WEBHOOK_URL}")
print(f"[DEBUG] Body (first 100 chars): {body_str[:100]}...")
try:
is_valid = verify_signature(
request_body=body_str,
signature_header=signature_header,
signature_key=SQUARE_WEBHOOK_SIGNATURE_KEY,
notification_url=SQUARE_WEBHOOK_URL,
)
if not is_valid:
print(
f"[ERROR] Invalid webhook signature from {request.client.host}"
)
print(f"[ERROR] This usually means:")
print(
f"[ERROR] 1. SQUARE_WEBHOOK_URL doesn't match Square Dashboard exactly"
)
print(f"[ERROR] 2. SQUARE_WEBHOOK_SIGNATURE_KEY is incorrect")
print(f"[ERROR] 3. Request body was modified in transit")
return JSONResponse(
status_code=status.HTTP_401_UNAUTHORIZED,
content={"error": "Invalid signature"},
)
print(f"[INFO] ✓ Webhook signature verified from {request.client.host}")
except Exception as e:
print(f"[ERROR] Signature verification exception: {e}")
return JSONResponse(
status_code=status.HTTP_401_UNAUTHORIZED,
content={"error": f"Signature verification failed: {str(e)}"},
)
else:
print(
f"[WARN] Webhook signature verification DISABLED - configure SQUARE_WEBHOOK_SIGNATURE_KEY"
)
# Parse JSON data
try:
data = json.loads(body_str)
except json.JSONDecodeError as e:
print(f"[ERROR] Invalid JSON from {request.client.host}: {e}")
return JSONResponse(
status_code=status.HTTP_400_BAD_REQUEST,
content={"error": "Invalid JSON"},
)
# Check idempotency (event_id)
event_id = data.get("event_id")
if event_id:
if event_id in processed_event_ids:
print(
f"[INFO] Duplicate event {event_id} - already processed (idempotency)"
)
return JSONResponse(
status_code=status.HTTP_200_OK,
content={
"status": "success",
"message": "Event already processed",
"event_id": event_id,
},
)
# Add to processed set
processed_event_ids.add(event_id)
# Prevent memory growth
if len(processed_event_ids) > MAX_STORED_EVENT_IDS:
processed_event_ids.pop()
# Get event info for quick response
webhook_type = data.get("type", "unknown")
event_id_val = data.get("event_id", "none")
merchant_id = data.get("merchant_id", "unknown")
# RESPOND IMMEDIATELY - Square requires fast response (< 2 seconds)
response_data = {
"status": "success",
"message": "Webhook received",
"event_id": event_id_val,
}
# Process webhook in background (async task)
asyncio.create_task(
process_webhook(
webhook_type, event_id_val, merchant_id, data, request.client.host
)
)
# Return immediate response
print(f"[INFO] 200 OK response sent for event {event_id_val}")
return JSONResponse(status_code=status.HTTP_200_OK, content=response_data)
except Exception as e:
# Catch-all for any unexpected errors - still respond quickly
print(f"[ERROR] Unexpected error in webhook handler: {e}")
import traceback
traceback.print_exc()
return JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={"error": "Internal server error"},
)
# Return immediate response
return JSONResponse(status_code=status.HTTP_200_OK, content=response_data)
async def process_webhook(
webhook_type: str,
event_id: str,
merchant_id: str,
data: Dict[str, Any],
client_ip: str,
):
"""
Process webhook data asynchronously after responding to Square
This runs in the background and doesn't block the HTTP response
"""
try:
print("\n" + "=" * 60)
print(f"[WEBHOOK] Received at {asyncio.get_event_loop().time()}")
print(f"[WEBHOOK] From: {client_ip}")
print(f"[WEBHOOK] Type: {webhook_type}")
print(f"[WEBHOOK] Event ID: {event_id}")
print(f"[WEBHOOK] Merchant: {merchant_id}")
print(f"[WEBHOOK] Full payload:")
print(json.dumps(data, indent=2))
print("=" * 60 + "\n")
# Process based on webhook type
if webhook_type == "inventory.count.updated":
print(f"[PROCESS] Processing inventory count update...")
inventory_counts = (
data.get("data", {}).get("object", {}).get("inventory_counts", [])
)
# Import here to avoid circular imports
import src.asquare as asquare
import src.glue as glue
for count in inventory_counts:
variation_id = count.get("catalog_object_id")
quantity = count.get("quantity")
state = count.get("state")
print(
f"[PROCESS] Catalog: {variation_id}, Qty: {quantity}, State: {state}"
)
try:
# Get parent item from variation ID
item = await asquare.get_item_by_variation_id(variation_id)
item_id = item.get("id")
item_name = item.get("name", "")
category_id = item.get("category_id")
print(f"[PROCESS] Item: {item_id} - {item_name}")
print(f"[PROCESS] Category: {category_id}")
# Only process LEGO category items
if category_id == SQUARE_LEGO_CAT_ID:
print(
f"[PROCESS] ✓ LEGO item detected - triggering enrichment"
)
# Trigger enrichment in background with item data to avoid redundant lookup
asyncio.create_task(
glue.enrich_single_item(item_id, item_data=item)
)
else:
print(f"[PROCESS] ✗ Not a LEGO item - skipping enrichment")
except Exception as e:
print(f"[ERROR] Failed to process variation {variation_id}: {e}")
elif webhook_type == "catalog.version.updated":
print(f"[PROCESS] Processing catalog version update...")
# Import here to avoid circular imports
import src.asquare as asquare
import src.glue as glue
catalog_object = data.get("data", {}).get("object", {})
object_type = catalog_object.get("type")
if object_type == "ITEM":
item_id = catalog_object.get("id")
print(f"[PROCESS] Item updated: {item_id}")
try:
# Get item details to check category
item = await asquare.get_item_by_id(item_id)
item_name = item.get("name", "")
category_id = item.get("category_id")
print(f"[PROCESS] Item: {item_id} - {item_name}")
print(f"[PROCESS] Category: {category_id}")
# Only process LEGO category items
if category_id == SQUARE_LEGO_CAT_ID:
print(
f"[PROCESS] ✓ LEGO item detected - triggering enrichment"
)
# Trigger enrichment in background with item data to avoid redundant lookup
asyncio.create_task(
glue.enrich_single_item(item_id, item_data=item)
)
else:
print(f"[PROCESS] ✗ Not a LEGO item - skipping enrichment")
except Exception as e:
print(f"[ERROR] Failed to process item {item_id}: {e}")
else:
print(f"[PROCESS] Object type '{object_type}' - skipping")
else:
print(f"[PROCESS] Webhook type '{webhook_type}' - no handler configured")
print(f"[PROCESS] ✓ Webhook processing complete for {event_id}")
except Exception as e:
print(f"[ERROR] Failed to process webhook {event_id}: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
# For development/testing - use uvicorn directly
import uvicorn
print("\n" + "=" * 60)
print("Studspider webhook boot dev")
print("=" * 60)
print(
f"Signature verification: {'ENABLED' if SQUARE_WEBHOOK_SIGNATURE_KEY else 'DISABLED'}"
)
print(f"Webhook URL: {SQUARE_WEBHOOK_URL or 'NOT CONFIGURED'}")
print("\nFor production, use Gunicorn:")
print(
" gunicorn conf/studspider:app -w 4 -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:7433"
)
print("=" * 60 + "\n")
uvicorn.run(app, host="0.0.0.0", port=7433, log_level="info")
-438
View File
@@ -1,438 +0,0 @@
import argparse
import asyncio
import json
import ssl
from aiohttp import web
from base64 import b64decode
from typing import Dict, Any
import uuid
import dotenv
import os
import re
import logging
from logging.handlers import RotatingFileHandler
from pathlib import Path
import banner
import get_sq_env
from time import time
time = int(time())
import asquare as sq
# Square SDK webhook validation
from square.utils.webhooks_helper import verify_signature
# from asquareclass import AsyncSquareClient # type: ignore
from abricksetapi import AsyncBricksetAPI
print("Loading environment variables...")
dotenv.load_dotenv()
SQUARE_LEGO_CAT_ID = os.environ.get("SQUARE_LEGO_CAT_ID")
SQUARE_WEBHOOK_SIGNATURE_KEY = os.environ.get("SQUARE_WEBHOOK_SIGNATURE_KEY")
SQUARE_WEBHOOK_URL = os.environ.get("SQUARE_WEBHOOK_URL")
from asyncio_throttle.throttler import Throttler
# set up logger
print("Setting up logger...")
logger = logging.getLogger("studspider")
logger.setLevel(logging.DEBUG)
handler = RotatingFileHandler("./logs/studspider.log", maxBytes=125000, backupCount=5)
logger.addHandler(handler)
lego = AsyncBricksetAPI(debug=True, logger=logger)
# sq = AsyncSquareClient(logger=logger)
# logging.basicConfig(
# filename="./logs/studspider.log", encoding="utf-8", level=logging.DEBUG
# )
print("Setting up rate limiter...")
print("Doing RE things...")
UA = re.compile(r"^kopling\/?", re.IGNORECASE)
async def run_update():
"""
DEPRECATED: Use glue.enrich_bulk() instead
Legacy RUNUPDATE command handler - kept for backwards compatibility
but now delegates to the new glue.enrich_bulk() function
"""
from src import glue
await glue.enrich_bulk()
def valid_uuid(val):
try:
uuid.UUID(str(val))
return True
except ValueError:
return False
# In-memory store for processed event IDs (idempotency)
processed_event_ids = set()
MAX_STORED_EVENT_IDS = 10000 # Prevent memory growth
async def hook_handler(request: web.Request) -> web.Response:
"""Handle incoming JSON webhook POSTs on /hook endpoint"""
try:
current_time = int(asyncio.get_event_loop().time())
# Verify request method
if request.method != "POST":
logger.error(
f"{current_time}\tInvalid request method to /hook from %s",
request.remote,
)
return web.Response(status=405, text="Method not allowed. Use POST.")
# Read raw body for signature verification
body = await request.read()
body_str = body.decode("utf-8")
# Verify Square webhook signature using Square SDK
signature_header = request.headers.get("x-square-hmacsha256-signature", "")
if SQUARE_WEBHOOK_SIGNATURE_KEY and SQUARE_WEBHOOK_URL:
if not signature_header:
logger.error(
f"{current_time}\tMissing signature header from {request.remote}"
)
print(f"[@DEBUG] Missing x-square-hmacsha256-signature header")
return web.Response(status=401, text="Unauthorized: Missing signature")
# Use Square SDK's official validation function
is_valid = verify_signature(
request_body=body_str,
signature_header=signature_header,
signature_key=SQUARE_WEBHOOK_SIGNATURE_KEY,
notification_url=SQUARE_WEBHOOK_URL,
)
if not is_valid:
logger.error(f"{current_time}\tInvalid signature from {request.remote}")
print(f"[@DEBUG] Invalid webhook signature!")
return web.Response(status=401, text="Unauthorized: Invalid signature")
print(f"[@DEBUG] ✓ Webhook signature verified")
else:
print(
f"[@DEBUG] ⚠️ Webhook signature verification DISABLED (set SQUARE_WEBHOOK_SIGNATURE_KEY and SQUARE_WEBHOOK_URL in .env)"
)
# Parse JSON data
try:
data = json.loads(body_str)
except json.JSONDecodeError as e:
print(f"[@DEBUG] Invalid JSON to /hook from {request.remote}: {str(e)}")
logger.error(f"Invalid JSON to /hook from {request.remote}: {str(e)}")
return web.Response(status=400, text="Invalid JSON in request body")
# Check for idempotency (event_id)
event_id = data.get("event_id")
if event_id:
if event_id in processed_event_ids:
print(
f"[@DEBUG] Duplicate event {event_id} - already processed (idempotency)"
)
logger.info(
f"{current_time}\tDuplicate event {event_id} from {request.remote}"
)
return web.json_response(
{
"status": "success",
"message": "Event already processed (idempotency)",
"event_id": event_id,
}
)
# Add to processed set
processed_event_ids.add(event_id)
# Prevent memory growth - remove oldest if too many
if len(processed_event_ids) > MAX_STORED_EVENT_IDS:
processed_event_ids.pop()
# Get event info for quick response
webhook_type = data.get("type", "unknown")
event_id_val = data.get("event_id", "none")
# RESPOND IMMEDIATELY - Square requires fast response (< 2 seconds)
# Don't do any processing before responding!
response_data = {
"status": "success",
"message": "Webhook received",
"event_id": event_id_val,
}
# Log and process AFTER sending response (async background tasks)
async def log_and_process():
# Log the received webhook with detailed output
print(f"\n{'='*60}")
print(f"[@DEBUG] WEBHOOK RECEIVED at {current_time}")
print(f"[@DEBUG] From: {request.remote}")
print(f"[@DEBUG] Headers: {dict(request.headers)}")
print(f"[@DEBUG] Data type: {webhook_type}")
print(f"[@DEBUG] Event ID: {event_id_val}")
print(f"[@DEBUG] Full payload:")
print(json.dumps(data, indent=2))
print(f"{'='*60}\n")
logger.info(
f"{current_time}\tWebhook received from {request.remote}: {json.dumps(data)}"
)
# Process the webhook data
if webhook_type == "inventory.count.updated":
print(f"[@DEBUG] Processing inventory count update...")
# Extract inventory data
inventory_counts = (
data.get("data", {}).get("object", {}).get("inventory_counts", [])
)
for count in inventory_counts:
print(
f"[@DEBUG] Catalog Object: {count.get('catalog_object_id')}"
)
print(f"[@DEBUG] Quantity: {count.get('quantity')}")
print(f"[@DEBUG] State: {count.get('state')}")
# Start background processing (don't wait for it)
asyncio.create_task(log_and_process())
# Return response immediately
return web.json_response(response_data)
except json.JSONDecodeError as e:
error_msg = f"JSON decode error in webhook: {str(e)}"
print(f"[@DEBUG] {error_msg}")
logger.error(error_msg)
return web.Response(status=400, text=error_msg)
except Exception as e:
error_msg = f"Webhook handler error: {str(e)}"
print(f"[@DEBUG] {error_msg}")
print(f"[@DEBUG] Exception type: {type(e).__name__}")
import traceback
print(f"[@DEBUG] Traceback:\n{traceback.format_exc()}")
logger.error(f"{error_msg}\n{traceback.format_exc()}")
return web.Response(status=500, text="Internal server error")
async def request_handler(request: web.Request) -> web.Response:
"""Handle incoming POST requests with command processing"""
try:
# Verify request method
if request.method != "POST":
logger.error(f"{time}\tInvalid request method from %s at ", request.remote)
return web.Response(status=405, text="Method not allowed. Use POST.")
# Get JSON data from request
try:
data = await request.json()
except json.JSONDecodeError:
logger.error("Invalid JSON from %s", request.remote)
return web.Response(
status=400, text="{time()}\tInvalid JSON in request body"
)
# Validate Idempotency-Key header
if valid_uuid(request.headers.get("Idempotency-Key")) is False:
logger.error("{time()}\tInvalid Idempotency-Key from %s", request.remote)
return web.Response(status=412, text="412 Headers rejected!")
# Validate User-Agent header
if not re.match(UA, str(request.headers.get("User-Agent"))):
logger.error("{time()}\tInvalid User-Agent from %s", request.remote)
return web.Response(status=412, text="412 Headers rejected!")
# Validate command exists
if "CMD" not in data:
logger.error("{time()}\tMissing CMD field from %s", request.remote)
return web.Response(status=400, text="Missing CMD field in request")
# Decode and process command
try:
decoded_cmd = b64decode(data["CMD"]).decode("utf-8")
print(
f"""At: {time}\nReceived command: {decoded_cmd}\nUser-Agent: {request.headers.get('User-Agent')}\nIdempotency-Key: {request.headers.get('Idempotency-Key')}"""
)
logging.info(
f"{time}\tCommand: {decoded_cmd}\tFrom: {request.remote}\tUA: {request.headers.get('User-Agent')}\tIdempotency-Key: {request.headers.get('Idempotency-Key')}"
)
# Process the command
if decoded_cmd == "RUNUPDATE":
response_data = {"status": "success", "message": "Update started"}
await run_update()
logging.info(f"{time}\tUpdate command processed successfully")
return web.json_response(response_data)
else:
return web.Response(status=400, text=f"Unknown command: {decoded_cmd}")
except Exception as e:
print(f"Error processing command: {str(e)}")
return web.Response(status=500, text=f"Server error: {str(e)}")
except Exception as e:
print(f"Request handler error: {str(e)}")
return web.Response(status=500, text="Internal server error")
print("Starting web server...")
os.system("cls" if os.name == "nt" else "clear")
app = web.Application()
app.router.add_post("/api", request_handler)
app.router.add_post("/hook", hook_handler)
parser = argparse.ArgumentParser(
description="studspider - LEGO Inventory Sync Service for Square POS and Brickset",
epilog="""
Examples:
# Basic HTTP server (recommended with nginx reverse proxy)
python src/test_server.py --host 0.0.0.0 --port 7432
# With separate webhook port
python src/test_server.py --port 7432 --hook-port 8080
# HTTPS using certificates from .env (set SSL_CERT_PATH and SSL_KEY_PATH)
python src/test_server.py --hook-port 8443 --hook-ssl-only
# HTTPS with command line override (self-signed cert for testing)
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only
# Full HTTPS (both endpoints)
python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443
# Production behind nginx (no SSL needed in app)
python src/test_server.py --host 127.0.0.1 --port 7432
Environment Variables (.env):
SSL_CERT_PATH - Default SSL certificate path
SSL_KEY_PATH - Default SSL key path
For more information, see doc/
""",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--host",
type=str,
default="localhost",
help="IP address to listen on (default: localhost, use 0.0.0.0 for all interfaces)",
)
parser.add_argument(
"--port", type=int, default=7432, help="Port for main API endpoint (default: 7432)"
)
parser.add_argument(
"--hook-port",
type=int,
default=None,
help="Separate port for webhook endpoint (default: None - uses same port as --port)",
)
parser.add_argument(
"--ssl-cert",
type=str,
default=get_sq_env.SSL_CERT_PATH,
help="Path to SSL certificate file in PEM format (default: from .env SSL_CERT_PATH)",
)
parser.add_argument(
"--ssl-key",
type=str,
default=get_sq_env.SSL_KEY_PATH,
help="Path to SSL private key file in PEM format (default: from .env SSL_KEY_PATH)",
)
parser.add_argument(
"--hook-ssl-only",
action="store_true",
help="Enable HTTPS only for webhook endpoint, main API stays HTTP (requires --ssl-cert and --ssl-key)",
)
args = parser.parse_args()
if __name__ == "__main__":
print("Creating log directory if it doesn't exist...")
os.makedirs("./logs/", exist_ok=True)
print(banner.generate("studspider v0.1"))
print("Built for PLAY-WELL TEKNOLOGIES by contact@studspider.sh\n\n")
# Setup SSL context if certificates provided
ssl_context = None
if args.ssl_cert and args.ssl_key:
# Check if trying to use P7B file
if args.ssl_cert.endswith(".p7b") or args.ssl_key.endswith(".p7b"):
print("ERROR: Cannot use P7B files directly for HTTPS server!")
print("\nP7B files are for Square's webhook signature verification only.")
print("Your server needs PEM format files (.pem)")
print("\nWorkflow:")
print("1. Generate PEM files: python generate_cert.py")
print("2. Convert to P7B for Square: python convert_to_p7b.py")
print("3. Upload P7B to Square Dashboard")
print("4. Run server with PEM files:")
print(
" python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443"
)
print("\nIf you have a P7B and need PEM:")
print(" python convert_from_p7b.py your_cert.p7b")
exit(1)
try:
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(args.ssl_cert, args.ssl_key)
print(f"✓ SSL enabled with cert: {args.ssl_cert}")
except FileNotFoundError as e:
print(f"ERROR: Certificate file not found: {e}")
exit(1)
except ssl.SSLError as e:
print(f"ERROR: Invalid certificate or key file: {e}")
print("\nMake sure you're using PEM format files.")
exit(1)
elif args.ssl_cert or args.ssl_key:
print("ERROR: Both --ssl-cert and --ssl-key must be provided together")
exit(1)
else:
print("ℹ️ Running without SSL (HTTP only)")
print(" For production with nginx reverse proxy, this is recommended.")
print(" Nginx will handle HTTPS, forward HTTP to this server.")
# If hook-port is specified, run a separate server for webhooks
if args.hook_port:
protocol_main = "https" if (ssl_context and not args.hook_ssl_only) else "http"
protocol_hook = "https" if ssl_context else "http"
# Create separate app for webhook
hook_app = web.Application()
hook_app.router.add_post("/hook", hook_handler)
# Run both servers concurrently
async def run_servers():
runner1 = web.AppRunner(app)
await runner1.setup()
# Main server uses SSL only if not hook-ssl-only mode
main_ssl = None if args.hook_ssl_only else ssl_context
site1 = web.TCPSite(runner1, args.host, args.port, ssl_context=main_ssl)
runner2 = web.AppRunner(hook_app)
await runner2.setup()
# Webhook server always uses SSL if certificates provided
site2 = web.TCPSite(
runner2, args.host, args.hook_port, ssl_context=ssl_context
)
await site1.start()
await site2.start()
print(f"Starting main server on {args.host}:{args.port} ({protocol_main})")
print(
f"Starting webhook server on {args.host}:{args.hook_port} ({protocol_hook})"
)
# Keep running
await asyncio.Event().wait()
asyncio.run(run_servers())
else:
protocol = "https" if ssl_context else "http"
print(f"Server running on {args.host}:{args.port} ({protocol})")
print(f"Webhook endpoint: {protocol}://{args.host}:{args.port}/hook")
web.run_app(app, host=args.host, port=args.port, ssl_context=ssl_context)
-51
View File
@@ -1,51 +0,0 @@
"""
Simple test script to verify Square webhook signature verification works correctly.
This uses the same test data from Square's documentation to ensure our implementation
matches their expected behavior.
"""
import sys
sys.path.insert(0, "src")
from square.utils.webhooks_helper import verify_signature
# Test data from Square's documentation
TEST_BODY = '{"hello":"world"}'
TEST_SIGNATURE = "2kRE5qRU2tR+tBGlDwMEw2avJ7QM4ikPYD/PJ3bd9Og="
TEST_SIGNATURE_KEY = "asdf1234"
TEST_NOTIFICATION_URL = "https://example.com/webhook"
def test_signature_validation():
"""Test that signature validation works with Square SDK"""
print("Testing Square webhook signature validation...")
print(f"Body: {TEST_BODY}")
print(f"Signature: {TEST_SIGNATURE}")
print(f"Key: {TEST_SIGNATURE_KEY}")
print(f"URL: {TEST_NOTIFICATION_URL}")
print()
# This should return True with Square's test data
is_valid = verify_signature(
request_body=TEST_BODY,
signature_header=TEST_SIGNATURE,
signature_key=TEST_SIGNATURE_KEY,
notification_url=TEST_NOTIFICATION_URL,
)
if is_valid:
print("✅ SUCCESS: Signature validation passed!")
print("The webhook implementation is using Square SDK correctly.")
return True
else:
print("❌ FAILURE: Signature validation failed!")
print("Something is wrong with the signature verification.")
return False
if __name__ == "__main__":
success = test_signature_validation()
sys.exit(0 if success else 1)
-102
View File
@@ -1,102 +0,0 @@
"""
Test script for webhook endpoint
Sends the Square inventory.count.updated webhook payload
"""
import asyncio
import aiohttp
import json
import ssl
import sys
async def test_webhook(verify_ssl=True):
url = "https://studspider.sh:8443/hook"
# Your Square webhook payload
payload = {
"merchant_id": "6SSW7HV8K2ST5",
"type": "inventory.count.updated",
"event_id": "df5f3813-a913-45a1-94e9-fdc3f7d5e3b6",
"created_at": "2019-10-29T18:38:45.455006797Z",
"data": {
"type": "inventory_counts",
"id": "84e4ac73-d605-4dbd-a9e5-ffff794ddb9d",
"object": {
"inventory_counts": [
{
"calculated_at": "2019-10-29T18:38:45.10296Z",
"catalog_object_id": "FGQ5JJWT2PYTHF35CKZ2DSKP",
"catalog_object_type": "ITEM_VARIATION",
"location_id": "YYQR03DGCTXA4",
"quantity": "10",
"state": "IN_STOCK",
}
]
},
},
}
print("Sending webhook test payload...")
print(f"URL: {url}")
print(
f"SSL Verification: {'Enabled' if verify_ssl else 'Disabled (accepting self-signed certs)'}"
)
print(json.dumps(payload, indent=2))
print("\n" + "=" * 60 + "\n")
# Create SSL context that allows self-signed certificates if verify_ssl is False
ssl_context = None
if not verify_ssl:
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
try:
connector = aiohttp.TCPConnector(ssl=ssl_context)
async with aiohttp.ClientSession(connector=connector) as session:
async with session.post(url, json=payload) as resp:
print(f"Status: {resp.status}")
print(f"Response headers: {dict(resp.headers)}")
response_text = await resp.text()
print(f"\nResponse body:")
try:
response_json = json.loads(response_text)
print(json.dumps(response_json, indent=2))
except json.JSONDecodeError:
print(response_text)
except aiohttp.ClientConnectorError as e:
print(f"Connection error: {e}")
print("\nTroubleshooting:")
print("1. Make sure the server is running")
print("2. Check if the hostname and port are correct")
print("3. For self-signed certificates, the script now accepts them by default")
print("\nServer start command:")
print(
" python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only"
)
except aiohttp.ServerTimeoutError as e:
print(f"Timeout error: {e}")
print("The server took too long to respond")
except Exception as e:
print(f"Error: {type(e).__name__}: {e}")
if __name__ == "__main__":
# Check command line arguments
verify_ssl = True
if len(sys.argv) > 1 and sys.argv[1] in ["--no-verify", "--insecure", "-k"]:
verify_ssl = False
print("⚠️ SSL verification disabled - accepting self-signed certificates\n")
elif len(sys.argv) > 1:
print("Usage: python src/test_webhook.py [--no-verify]")
print(
" --no-verify: Disable SSL certificate verification (for self-signed certs)"
)
sys.exit(1)
# Default to no verification since we're likely using self-signed certs
asyncio.run(test_webhook(verify_ssl=False))
Generated
-1193
View File
File diff suppressed because it is too large. Load diff