renamert

.env.example

@@ -0,0 +1,72 @@
+# studspider Environment Configuration
+
+# ============================================================================
+# Square API Configuration
+# ============================================================================
+
+# Environment: "production" or "sandbox"
+SQUARE_ENV=sandbox
+
+# Sandbox credentials
+SANDBOX_SQUARE_APP_ID=your_sandbox_app_id
+SANDBOX_SQUARE_TOKEN=your_sandbox_access_token
+
+# Production credentials
+PROD_SQUARE_APP_ID=your_production_app_id
+PROD_SQUARE_TOKEN=your_production_access_token
+
+# Square LEGO category ID (find this in Square Dashboard)
+SQUARE_LEGO_CAT_ID=your_lego_category_id
+
+# ============================================================================
+# Brickset API Configuration
+# ============================================================================
+
+BRICKSET_USERNAME=your_brickset_username
+BRICKSET_PASS=your_brickset_password
+BRICKSET_API_KEY=your_brickset_api_key
+
+# ============================================================================
+# SSL Certificate Configuration (Optional)
+# ============================================================================
+
+# NOTE: Only needed if running HTTPS directly (not recommended with nginx)
+# For nginx reverse proxy setup, leave these empty
+
+# Path to SSL certificate in PEM format
+# SSL_CERT_PATH=/path/to/cert.pem
+
+# Path to SSL private key in PEM format
+# SSL_KEY_PATH=/path/to/key.pem
+
+# ============================================================================
+# Square Webhook Certificate (P7B Format)
+# ============================================================================
+
+# Path to P7B certificate file for Square webhook signature verification
+# This file is UPLOADED to Square Dashboard, not used by your server
+# Generate with: python convert_to_p7b.py
+SQUARE_P7B_PATH=./cert.p7b
+
+# ============================================================================
+# Server Configuration (Optional)
+# ============================================================================
+
+# Default host and port (can override with command line args)
+# SERVER_HOST=0.0.0.0
+# SERVER_PORT=7432
+# SERVER_HOOK_PORT=8080
+
+# ============================================================================
+# Usage Examples
+# ============================================================================
+
+# Basic HTTP server (recommended with nginx):
+#   python src/test_server.py
+#
+# With SSL from .env:
+#   Set SSL_CERT_PATH and SSL_KEY_PATH above, then:
+#   python src/test_server.py --hook-port 8443 --hook-ssl-only
+#
+# Override SSL from command line:
+#   python src/test_server.py --ssl-cert /custom/cert.pem --ssl-key /custom/key.pem

.gitignore

@@ -35,3 +35,6 @@ assets/external/
 __pycache__/
 win/
 .github/
+bin/
+activate/
+build/

doc/CERTIFICATE_FORMATS.md

@@ -0,0 +1,136 @@
+# Understanding PEM vs P7B Certificates
+
+## Quick Summary
+
+**You CANNOT use P7B files to run your HTTPS server!**
+
+- **PEM files** = Run your server with HTTPS ✓
+- **P7B files** = Upload to Square Dashboard ✓
+
+## The Complete Picture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Your HTTPS Server                        │
+│                                                             │
+│  Uses: cert.pem + key.pem (PEM format)                    │
+│  Command: python src/test_server.py                        │
+│           --ssl-cert cert.pem                              │
+│           --ssl-key key.pem                                │
+│           --hook-port 8443                                 │
+└─────────────────────────────────────────────────────────────┘
+                         │
+                         │ HTTPS traffic
+                         ▼
+┌─────────────────────────────────────────────────────────────┐
+│                  Square's Servers                           │
+│                                                             │
+│  They verify your server using: cert.p7b                   │
+│  (which YOU upload to Square Dashboard)                    │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Why Two Formats?
+
+### PEM Format (.pem files)
+- **Contains**: Certificate AND private key (separate files)
+- **Used by**: Your HTTPS server to encrypt traffic
+- **Files needed**: 
+  - `cert.pem` - Your public certificate
+  - `key.pem` - Your private key (NEVER share this!)
+
+### P7B Format (.p7b files)  
+- **Contains**: Certificate ONLY (no private key)
+- **Used by**: Square to verify webhook signatures
+- **File**: `cert.p7b` - Uploaded to Square Dashboard
+
+## Step-by-Step Workflow
+
+### 1. Generate Certificates (PEM format)
+```powershell
+python generate_cert.py
+```
+**Creates:**
+- `cert.pem` - Your certificate (public)
+- `key.pem` - Your private key (secret!)
+
+### 2. Start Your HTTPS Server (uses PEM)
+```powershell
+python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443 --hook-ssl-only
+```
+**Server runs at:**
+- `https://localhost:8443/hook`
+
+### 3. Convert Certificate to P7B for Square
+```powershell
+python convert_to_p7b.py
+```
+**Creates:**
+- `cert.p7b` - Certificate in P7B format (for Square)
+
+### 4. Upload P7B to Square
+1. Go to: https://developer.squareup.com/apps
+2. Your App → **Webhooks** → **Manage Notification Settings**
+3. Upload `cert.p7b`
+
+### 5. Configure Square Webhook URL
+- Enter: `https://yourdomain.com:8443/hook`
+- Square will verify using the P7B you uploaded
+
+## Common Mistakes
+
+### ❌ Trying to use P7B with your server
+```powershell
+# THIS WON'T WORK!
+python src/test_server.py --ssl-cert cert.p7b --ssl-key key.p7b
+```
+**Error:** P7B doesn't contain the private key needed for HTTPS!
+
+### ✓ Correct usage
+```powershell
+# Server uses PEM files
+python src/test_server.py --ssl-cert cert.pem --ssl-key key.pem --hook-port 8443
+
+# P7B file is uploaded to Square Dashboard (not used by your server)
+```
+
+## If You Only Have a P7B File
+
+If someone gave you a P7B file, you can extract the certificate:
+
+```powershell
+python convert_from_p7b.py your_cert.p7b --output cert.pem
+```
+
+**BUT:** You still need the private key (`key.pem`) to run HTTPS!
+
+If you don't have the private key, you must:
+1. Generate a new certificate pair: `python generate_cert.py`
+2. Or get the private key from whoever created the P7B
+
+## File Checklist
+
+To run your HTTPS server with Square webhooks, you need:
+
+- [ ] `cert.pem` - Certificate (PEM format) - For your server
+- [ ] `key.pem` - Private key (PEM format) - For your server  
+- [ ] `cert.p7b` - Certificate (P7B format) - Upload to Square
+
+Generate all three with:
+```powershell
+python generate_cert.py      # Creates cert.pem and key.pem
+python convert_to_p7b.py      # Creates cert.p7b from cert.pem
+```
+
+## Security Notes
+
+- **Never share `key.pem`** - This is your private key!
+- **`cert.pem` is public** - Safe to share
+- **`cert.p7b` is public** - You upload this to Square
+- Self-signed certificates are fine for testing but use Let's Encrypt for production
+
+## Need Help?
+
+See the detailed guides:
+- `HTTPS_SETUP.md` - Complete HTTPS setup instructions
+- `SQUARE_WEBHOOK_SETUP.md` - Square webhook configuration

doc/HTTPS_SETUP.md

@@ -9,6 +9,14 @@ python generate_cert.py
 
 This creates `cert.pem` and `key.pem` files.
 
+### 1.5. Convert to P7B for Square (Required for Square Webhooks)
+```powershell
+python convert_to_p7b.py
+```
+
+This creates `cert.p7b` which Square needs for webhook signature verification.
+Upload this P7B file to Square Developer Dashboard → Webhooks → Manage Notification Settings.
+
 ### 2. Run Server with HTTPS
 
 #### Option A: HTTPS for webhook only (recommended for development)
@@ -115,6 +123,39 @@ python src/test_server.py
 
 ## Square Webhook Configuration
 
+### Certificate Requirements
+Square requires a **P7B (PKCS#7)** certificate file for webhook signature verification.
+
+### Setup Steps:
+
+1. **Generate your certificate** (if you haven't already):
+   ```powershell
+   python generate_cert.py
+   ```
+
+2. **Convert PEM to P7B format**:
+   ```powershell
+   # Basic conversion
+   python convert_to_p7b.py
+   
+   # With intermediate certificates (production)
+   python convert_to_p7b.py --cert fullchain.pem --chain intermediate.pem --output square.p7b
+   ```
+
+3. **Upload to Square Dashboard**:
+   - Go to https://developer.squareup.com/
+   - Navigate to your application
+   - Click on "Webhooks"
+   - Click "Manage Notification Settings"
+   - Upload the `cert.p7b` file
+   - Square will use this to verify webhook signatures
+
+4. **Configure webhook URL in Square**:
+   - Use your public HTTPS URL: `https://yourdomain.com:8443/hook`
+   - Square requires valid SSL (not self-signed for production)
+
+### Testing with Square Webhooks:
+
 When setting up webhooks in Square Dashboard:
 1. Use your HTTPS URL: `https://yourdomain.com:8443/hook`
 2. Square requires valid SSL (not self-signed)

doc/NGINX_SETUP.md

@@ -0,0 +1,356 @@
+# Nginx Reverse Proxy Configuration for studspider
+
+## Overview
+
+When using nginx as a reverse proxy:
+- **Nginx handles HTTPS** (SSL/TLS termination)
+- **studspider runs on HTTP** internally (no SSL needed)
+- **nginx forwards requests** to studspider
+
+## Architecture
+
+```
+Internet (HTTPS) → nginx (443) → studspider (7432/HTTP)
+                      ↓
+            Handles SSL certificates
+            Handles HTTPS traffic
+            Forwards to HTTP backend
+```
+
+## Benefits
+
+✓ **No SSL certificates needed in studspider**
+✓ **nginx handles SSL efficiently**
+✓ **Easy certificate renewal** (Let's Encrypt)
+✓ **Better performance** (nginx is optimized for this)
+✓ **Additional security** (rate limiting, IP filtering, etc.)
+
+---
+
+## Configuration Examples
+
+### Basic Configuration (HTTP → HTTPS redirect + webhook proxy)
+
+Create `/etc/nginx/sites-available/studspider`:
+
+```nginx
+# Redirect HTTP to HTTPS
+server {
+    listen 80;
+    server_name studspider.sh www.studspider.sh;
+    
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+# HTTPS server
+server {
+    listen 443 ssl http2;
+    server_name studspider.sh www.studspider.sh;
+
+    # SSL certificates (Let's Encrypt)
+    ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
+    
+    # SSL configuration (modern, secure)
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers HIGH:!aNULL:!MD5;
+    ssl_prefer_server_ciphers on;
+
+    # Logs
+    access_log /var/log/nginx/studspider_access.log;
+    error_log /var/log/nginx/studspider_error.log;
+
+    # Webhook endpoint (Square webhooks)
+    location /hook {
+        proxy_pass http://localhost:7432/hook;
+        
+        # Forward headers
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        
+        # Timeouts
+        proxy_connect_timeout 60s;
+        proxy_send_timeout 60s;
+        proxy_read_timeout 60s;
+        
+        # Disable buffering for real-time responses
+        proxy_buffering off;
+    }
+
+    # Main API endpoint (optional)
+    location / {
+        proxy_pass http://localhost:7432/;
+        
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}
+```
+
+### Separate Port Configuration (webhook on different port)
+
+```nginx
+server {
+    listen 443 ssl http2;
+    server_name studspider.sh;
+
+    ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
+
+    # Main API
+    location / {
+        proxy_pass http://localhost:7432/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
+
+# Separate server for webhooks on different port
+server {
+    listen 8443 ssl http2;
+    server_name studspider.sh;
+
+    ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
+
+    # Webhook only
+    location /hook {
+        proxy_pass http://localhost:8080/hook;  # studspider webhook port
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
+```
+
+### With Rate Limiting (recommended for webhooks)
+
+```nginx
+# Define rate limit zone (outside server block)
+limit_req_zone $binary_remote_addr zone=webhook_limit:10m rate=10r/s;
+
+server {
+    listen 443 ssl http2;
+    server_name studspider.sh;
+
+    ssl_certificate /etc/letsencrypt/live/studspider.sh/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/studspider.sh/privkey.pem;
+
+    # Webhook with rate limiting
+    location /hook {
+        # Rate limit: 10 requests per second per IP
+        limit_req zone=webhook_limit burst=20 nodelay;
+        
+        # Only allow POST requests
+        limit_except POST {
+            deny all;
+        }
+        
+        proxy_pass http://localhost:7432/hook;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
+```
+
+---
+
+## Setup Steps
+
+### 1. Install nginx (if not already installed)
+
+```bash
+# Ubuntu/Debian
+sudo apt update
+sudo apt install nginx
+
+# CentOS/RHEL
+sudo yum install nginx
+```
+
+### 2. Get SSL Certificate (Let's Encrypt - Free!)
+
+```bash
+# Install certbot
+sudo apt install certbot python3-certbot-nginx
+
+# Get certificate for your domain
+sudo certbot --nginx -d studspider.sh -d www.studspider.sh
+
+# Certbot will automatically:
+# 1. Generate certificate
+# 2. Configure nginx
+# 3. Set up auto-renewal
+```
+
+### 3. Create nginx Configuration
+
+```bash
+# Create config file
+sudo nano /etc/nginx/sites-available/studspider
+
+# Paste one of the configurations above
+
+# Create symbolic link to enable site
+sudo ln -s /etc/nginx/sites-available/studspider /etc/nginx/sites-enabled/
+
+# Test configuration
+sudo nginx -t
+
+# Reload nginx
+sudo systemctl reload nginx
+```
+
+### 4. Start studspider (HTTP only, no SSL needed!)
+
+```bash
+# Simple - single port
+python src/test_server.py --port 7432
+
+# Or with separate webhook port
+python src/test_server.py --port 7432 --hook-port 8080
+```
+
+### 5. Configure Square Webhook URL
+
+In Square Developer Dashboard:
+- **Webhook URL**: `https://studspider.sh/hook`
+- **Certificate**: Upload the P7B file (see below)
+
+### 6. Generate P7B for Square
+
+```bash
+# Convert your Let's Encrypt certificate to P7B for Square
+python convert_to_p7b.py --cert /etc/letsencrypt/live/studspider.sh/fullchain.pem --output square.p7b
+
+# Upload square.p7b to Square Dashboard
+```
+
+---
+
+## Systemd Service (Run studspider at boot)
+
+Create `/etc/systemd/system/studspider.service`:
+
+```ini
+[Unit]
+Description=studspider - LEGO Inventory Sync Service
+After=network.target
+
+[Service]
+Type=simple
+User=www-data
+WorkingDirectory=/opt/studspider
+Environment="PATH=/opt/studspider/.venv/bin"
+ExecStart=/opt/studspider/.venv/bin/python src/test_server.py --port 7432
+Restart=always
+RestartSec=10
+
+# Logging
+StandardOutput=append:/var/log/studspider/stdout.log
+StandardError=append:/var/log/studspider/stderr.log
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Enable and start:
+
+```bash
+# Create log directory
+sudo mkdir -p /var/log/studspider
+sudo chown www-data:www-data /var/log/studspider
+
+# Enable service
+sudo systemctl enable studspider
+
+# Start service
+sudo systemctl start studspider
+
+# Check status
+sudo systemctl status studspider
+
+# View logs
+sudo journalctl -u studspider -f
+```
+
+---
+
+## Testing
+
+### Test nginx configuration
+```bash
+sudo nginx -t
+```
+
+### Test studspider locally
+```bash
+curl http://localhost:7432/hook -X POST -H "Content-Type: application/json" -d '{"test":"data"}'
+```
+
+### Test through nginx
+```bash
+curl https://studspider.sh/hook -X POST -H "Content-Type: application/json" -d '{"test":"data"}'
+```
+
+### Monitor logs
+```bash
+# nginx logs
+sudo tail -f /var/log/nginx/studspider_access.log
+sudo tail -f /var/log/nginx/studspider_error.log
+
+# studspider logs
+tail -f ./logs/studspider.log
+```
+
+---
+
+## Troubleshooting
+
+### 502 Bad Gateway
+- Check if studspider is running: `ps aux | grep test_server.py`
+- Check the port matches nginx config
+- Check firewall isn't blocking internal connections
+
+### Certificate Errors
+- Ensure Let's Encrypt renewal is working: `sudo certbot renew --dry-run`
+- Check certificate files exist and have correct permissions
+
+### Webhook Not Receiving
+- Verify nginx is forwarding: Check nginx access logs
+- Verify studspider is receiving: Check studspider logs
+- Test locally first, then through nginx
+
+### Connection Refused
+- Check studspider is listening: `sudo netstat -tlnp | grep 7432`
+- Check nginx proxy_pass points to correct host:port
+
+---
+
+## Summary
+
+### With nginx reverse proxy:
+
+✓ **studspider runs simple HTTP** (no SSL certificates needed)
+```bash
+python src/test_server.py --port 7432
+```
+
+✓ **nginx handles HTTPS** (manages certificates automatically)
+
+✓ **Square gets HTTPS webhook** (`https://studspider.sh/hook`)
+
+✓ **You upload P7B to Square** (converted from nginx's certificate)
+
+### This is the recommended production setup!

src/scripts/convert_from_p7b.py

@@ -0,0 +1,110 @@
+"""
+Convert P7B (PKCS#7) certificate to PEM format for use with the server
+Use this if you have a P7B file and need to extract PEM certificates
+"""
+
+import subprocess
+import sys
+import os
+
+
+def convert_p7b_to_pem(p7b_file, cert_output="cert.pem", extract_key=False):
+    """
+    Convert P7B certificate to PEM format
+
+    Args:
+        p7b_file: Input P7B (PKCS#7) file
+        cert_output: Output PEM certificate file
+        extract_key: If True, try to extract private key (rarely works with P7B)
+    """
+
+    # Check if OpenSSL is available
+    try:
+        subprocess.run(["openssl", "version"], capture_output=True, check=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("ERROR: OpenSSL not found!")
+        print("\nFor Windows, you can:")
+        print(
+            "1. Install OpenSSL from: https://slproweb.com/products/Win32OpenSSL.html"
+        )
+        print("2. Or use Git Bash which includes OpenSSL")
+        print("3. Or use WSL (Windows Subsystem for Linux)")
+        return False
+
+    # Check if input file exists
+    if not os.path.exists(p7b_file):
+        print(f"ERROR: P7B file not found: {p7b_file}")
+        print(f"Current directory: {os.getcwd()}")
+        return False
+
+    print(f"Converting {p7b_file} to PEM format...")
+
+    # Convert P7B to PEM
+    cmd = ["openssl", "pkcs7", "-print_certs", "-in", p7b_file, "-out", cert_output]
+
+    try:
+        result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+
+        if os.path.exists(cert_output):
+            print(f"\n✓ PEM certificate created: {os.path.abspath(cert_output)}")
+            print(f"  File size: {os.path.getsize(cert_output)} bytes")
+
+            # Check what was extracted
+            with open(cert_output, "r") as f:
+                content = f.read()
+                cert_count = content.count("BEGIN CERTIFICATE")
+                print(f"  Certificates found: {cert_count}")
+
+            print("\n" + "=" * 60)
+            print("IMPORTANT:")
+            print("=" * 60)
+            print("⚠️  P7B files usually contain ONLY certificates, not private keys!")
+            print("\nTo run an HTTPS server, you need:")
+            print("1. Certificate (cert.pem) - ✓ Extracted")
+            print("2. Private Key (key.pem) - ✗ NOT in P7B file")
+            print("\nIf you don't have the private key:")
+            print("- Generate a new certificate pair: python generate_cert.py")
+            print("- Or obtain the private key from your certificate provider")
+            print("=" * 60)
+
+            return True
+        else:
+            print("ERROR: Output file was not created")
+            return False
+
+    except subprocess.CalledProcessError as e:
+        print(f"ERROR: Conversion failed: {e}")
+        if e.stderr:
+            print(f"OpenSSL error: {e.stderr}")
+        return False
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Convert P7B (PKCS#7) certificate to PEM format"
+    )
+    parser.add_argument("p7b_file", help="Input P7B certificate file")
+    parser.add_argument(
+        "--output",
+        default="cert.pem",
+        help="Output PEM certificate file (default: cert.pem)",
+    )
+
+    args = parser.parse_args()
+
+    print("=" * 60)
+    print("P7B to PEM Certificate Converter")
+    print("=" * 60)
+    print()
+
+    success = convert_p7b_to_pem(args.p7b_file, args.output)
+
+    if success:
+        print("\n✓ Conversion complete!")
+        print("\nNext steps:")
+        print("1. If you have the private key separately, use it with --ssl-key")
+        print("2. If not, generate a new certificate: python generate_cert.py")
+    else:
+        sys.exit(1)