commit 6c21525b79f309d232a751b07b7217bc06802ce5 Author: D Stephenson Date: Wed Apr 22 20:43:59 2026 +0000 Initial commit Co-Authored-By: Claude Sonnet 4.6 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f4f1a84 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,40 @@ +# ── RV50x Template Manager — Docker ignore ────────────────────────────────── +# Files and folders that should NOT be copied into the Docker image. +# Data directories are mounted as volumes instead. + +# Virtual environment — dependencies are installed fresh in the image +.venv/ + +# Data directories — these are bind-mounted as named volumes +template_downloads/ +template_uploads/ +xml_templates/ + +# Legacy and backup scripts — not needed in the container +download_csv.py +upload_csv.py +rv50x_template_manager.py +modems.csv + +# Standalone CLI scripts — the web app (app.py) replaces these in Docker +download.py +upload.py + +# Reports — live in the volumes, not the image +report_*.txt + +# Python cache +__pycache__/ +*.pyc +*.pyo +*.pyd + +# Environment files — never bake secrets into the image +.env +.env.* + +# Editor and OS files +.DS_Store +.vscode/ +*.swp +*.swo diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0e4b1f5 --- /dev/null +++ b/.env.example @@ -0,0 +1,51 @@ +# ── RV50x Template Manager — Environment Configuration ───────────────────── +# +# Copy this file to .env and fill in your values. +# Never commit .env to version control — it contains secrets. +# +# docker-compose reads this file automatically. +# For Portainer: paste these into the Environment Variables section of the stack. +# ─────────────────────────────────────────────────────────────────────────── + + +# ── NocoDB connection ──────────────────────────────────────────────────────── +# +# If using the built-in NocoDB from this stack: +# NOCODB_URL=http://nocodb:8080 +# +# If pointing at an existing external NocoDB instance: +# NOCODB_URL=http://192.168.16.130:8080 +# +# After spinning up the stack and importing your data, update the IDs below +# by reading them from the NocoDB browser URL: +# http://host/w98wg3nt/{NOCODB_BASE_ID}/{NOCODB_TABLE_ID}/{NOCODB_VIEW_ID}/... + +NOCODB_URL=http://nocodb:8080 +NOCODB_TOKEN=your-nocodb-api-token-here +NOCODB_BASE_ID=your-base-id-here +NOCODB_TABLE_ID=your-table-id-here +NOCODB_VIEW_ID=your-view-id-here + + +# ── PostgreSQL password ────────────────────────────────────────────────────── +# Used internally by NocoDB. Choose a strong password. +# You won't need to type this anywhere — it's only used container-to-container. + +POSTGRES_PASSWORD=changeme_use_a_strong_password_here + + +# ── NocoDB JWT secret ──────────────────────────────────────────────────────── +# Used to sign NocoDB auth tokens. Any long random string works. +# Generate one with: openssl rand -hex 32 + +NC_JWT_SECRET=changeme_use_a_long_random_string_here + + +# ── Playwright timeouts (optional — defaults shown) ────────────────────────── +# Increase these if your modems are slow to respond. +# Values are in milliseconds. + +# PAGE_TIMEOUT=90000 +# DOWNLOAD_TIMEOUT=120000 +# UPLOAD_TIMEOUT=120000 +# MAX_RETRIES=3 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4b79735 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +certs/ +__pycache__/ +*.pyc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f57876b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +# ── RV50x Template Manager ───────────────────────────────────────────────── +# Uses the official Playwright Python image which has Chromium and all +# required system libraries pre-installed — no manual apt installs needed. +# ─────────────────────────────────────────────────────────────────────────── + +FROM mcr.microsoft.com/playwright/python:v1.44.0-jammy + +# Set working directory inside the container +WORKDIR /app + +# ── Install Python dependencies ───────────────────────────────────────────── +# Copy requirements first so Docker caches this layer — only rebuilds when +# requirements.txt changes, not every time app code changes. +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# ── Install Playwright's Chromium browser ─────────────────────────────────── +# The base image has the system libs; this installs the actual browser binary. +RUN playwright install chromium + +# ── Copy application files ────────────────────────────────────────────────── +COPY app.py . +COPY index.html . + +# ── Create data directories ───────────────────────────────────────────────── +# These will be overridden by volume mounts in docker-compose, but we create +# them here so the app works even if volumes aren't configured. +RUN mkdir -p /data/template_downloads \ + /data/template_uploads \ + /data/xml_templates + +# ── Environment defaults ──────────────────────────────────────────────────── +# These are overridden by the .env file or docker-compose environment section. +ENV DOWNLOAD_DIR=/data/template_downloads +ENV UPLOAD_DIR=/data/template_uploads +ENV TEMPLATES_DIR=/data/xml_templates +ENV PYTHONUNBUFFERED=1 + +# ── Expose port ───────────────────────────────────────────────────────────── +# Change the left number in docker-compose.yml to remap to a different host port. +EXPOSE 8000 + +# ── Start the application ─────────────────────────────────────────────────── +# SSL_CERT and SSL_KEY env vars are optional — if set, uvicorn serves HTTPS. +# If not set, falls back to plain HTTP (useful for local dev). +CMD ["sh", "-c", "\ + if [ -n \"$SSL_CERT\" ] && [ -n \"$SSL_KEY\" ]; then \ + echo 'Starting with HTTPS'; \ + python -m uvicorn app:app --host 0.0.0.0 --port 8000 \ + --ssl-certfile \"$SSL_CERT\" --ssl-keyfile \"$SSL_KEY\"; \ + else \ + echo 'Starting with HTTP (no SSL vars set)'; \ + python -m uvicorn app:app --host 0.0.0.0 --port 8000; \ + fi"] diff --git a/README-docker.md b/README-docker.md new file mode 100644 index 0000000..46c0d80 --- /dev/null +++ b/README-docker.md @@ -0,0 +1,730 @@ +# RV50x Template Manager — Docker Edition + +A containerized web-based tool for managing configuration templates on Sierra Wireless AirLink RV50x modems. Runs as a full Docker stack including the web application, NocoDB, and PostgreSQL — start and stop the entire thing with a single command. + +--- + +## Table of Contents + +1. [Stack Overview](#stack-overview) +2. [Requirements](#requirements) +3. [Project Structure](#project-structure) +4. [First-Time Setup](#first-time-setup) +5. [The .env File](#the-env-file) +6. [Building and Starting](#building-and-starting) +7. [NocoDB Setup](#nocodb-setup) +8. [Updating the App](#updating-the-app) +9. [Managing the Stack](#managing-the-stack) +10. [Using Portainer](#using-portainer) +11. [Migrating to a New Machine](#migrating-to-a-new-machine) +12. [Reconnecting to a New NocoDB Instance](#reconnecting-to-a-new-nocodb-instance) +13. [Data and Volumes](#data-and-volumes) +14. [Troubleshooting](#troubleshooting) + +--- + +## Stack Overview + +The stack runs three containers: + +| Container | Image | Purpose | Default Port | +|---|---|---|---| +| `rv50x-manager` | Built from `Dockerfile` | FastAPI web app + Playwright | Your choice | +| `rv50x-nocodb` | `nocodb/nocodb:latest` | NocoDB UI and API | 8090 | +| `rv50x-postgres` | `postgres:16-alpine` | PostgreSQL database for NocoDB | Internal only | + +All three containers communicate over a private internal Docker network. PostgreSQL is never exposed to the host — only NocoDB can reach it. The web app talks to NocoDB via the internal hostname `nocodb`. + +``` +Your Browser + │ + ▼ +rv50x-manager (YOUR_PORT) ──→ nocodb (8090) ──→ postgres (internal) + │ + ▼ +Your Modems (port 443, via Playwright) +``` + +--- + +## Requirements + +- Docker Engine 24+ or Docker Desktop +- Docker Compose v2 (`docker compose` or `docker-compose`) +- Network access from the Docker host to your modems on port 443 +- Portainer (optional, but recommended for easy management) + +--- + +## Project Structure + +``` +/opt/rv50x-manager/ +├── app.py ← FastAPI backend +├── index.html ← Web UI +├── requirements.txt ← Python dependencies +├── Dockerfile ← Container build instructions +├── docker-compose.yml ← Stack definition +├── .env ← Your secrets and config (never commit this) +├── .env.example ← Template for .env +└── .dockerignore ← Files excluded from the Docker image +``` + +**Not needed in the Docker folder** (kept separately if you want CLI access): +- `download.py` / `upload.py` — standalone CLI scripts +- `modems.csv` — legacy device list +- `.venv/` — Python virtual environment + +--- + +## First-Time Setup + +### Step 1 — Install Docker + +**Fedora / RHEL / Rocky:** +```bash +sudo dnf install -y docker docker-compose-plugin +sudo systemctl start docker +sudo systemctl enable docker +sudo usermod -aG docker $USER # log out and back in after this +``` + +**Debian / Ubuntu:** +```bash +sudo apt-get install -y docker.io docker-compose-plugin +sudo systemctl start docker +sudo systemctl enable docker +sudo usermod -aG docker $USER +``` + +Verify Docker is working: +```bash +docker run hello-world +``` + +### Step 2 — Copy project files to the host machine + +Create the install directory and set permissions, then copy your files: + +```bash +# Create the directory and give your user ownership +sudo mkdir -p /opt/rv50x-manager +sudo chown $USER:$USER /opt/rv50x-manager + +# Copy all project files into it +cp app.py index.html requirements.txt Dockerfile \ + docker-compose.yml .env.example .dockerignore \ + /opt/rv50x-manager/ + +cd /opt/rv50x-manager +``` + +### Step 3 — Create your .env file + +```bash +cd /opt/rv50x-manager +cp .env.example .env +nano .env # or use any text editor +``` + +See [The .env File](#the-env-file) section below for what to put in each field. + +### Step 4 — Set your port + +Open `/opt/rv50x-manager/docker-compose.yml` and find this line under `rv50x-manager`: + +```yaml +ports: + - "YOUR_PORT:8000" +``` + +Replace `YOUR_PORT` with the port number you want to use on the host machine. For example: + +```yaml +ports: + - "8001:8000" +``` + +The app will then be accessible at `http://host-ip:8001`. + +### Step 5 — Build and start + +```bash +cd /opt/rv50x-manager +docker-compose up -d +``` + +This builds the `rv50x-manager` image and starts all three containers. The first build takes a few minutes because it downloads the Playwright base image and installs Chromium. + +### Step 6 — Set up NocoDB + +See [NocoDB Setup](#nocodb-setup) below. + +### Step 7 — Update .env with NocoDB IDs + +After importing your data into NocoDB, update `.env` with the real IDs and restart the manager: + +```bash +docker-compose restart rv50x-manager +``` + +### Step 8 — Open the app + +``` +http://your-docker-host-ip:YOUR_PORT +``` + +--- + +## The .env File + +Create this file by copying `.env.example` and filling in your values. It is read automatically by `docker-compose`. **Never commit this file to version control** — it contains passwords and API tokens. + +```bash +# ── NocoDB connection ────────────────────────────────────────────────────── +# +# Use "http://nocodb:8080" to connect to the NocoDB container in this stack. +# Use your external NocoDB URL if you prefer to point at an existing instance. +NOCODB_URL=http://nocodb:8080 + +# Your NocoDB API token. +# Get it from: NocoDB → Profile (bottom-left) → Team & Settings → API Tokens +# Tokens look like: eWU_ilelaCtNy1JzC7vf41DokkqFOovcLHM0zVml +NOCODB_TOKEN=your-api-token-here + +# These three IDs come from the NocoDB browser URL after you import your data. +# The URL structure is: +# http://host:8090/{org_id}/{BASE_ID}/{TABLE_ID}/{VIEW_ID}/table-name +# Leave as placeholder for now — update after NocoDB setup (Step 6 above). +NOCODB_BASE_ID=your-base-id-here +NOCODB_TABLE_ID=your-table-id-here +NOCODB_VIEW_ID=your-view-id-here + +# ── PostgreSQL ───────────────────────────────────────────────────────────── +# Internal password used between NocoDB and PostgreSQL containers only. +# You will never need to type this manually — make it long and strong. +POSTGRES_PASSWORD=SomeLongStrongPassword123! + +# ── NocoDB JWT secret ────────────────────────────────────────────────────── +# Any long random string used to sign NocoDB authentication tokens. +# Generate one with: openssl rand -hex 32 +NC_JWT_SECRET=paste-a-long-random-string-here + +# ── Playwright timeouts (optional) ──────────────────────────────────────── +# Uncomment and adjust if your modems are slow to respond. Values in ms. +# PAGE_TIMEOUT=90000 +# DOWNLOAD_TIMEOUT=120000 +# UPLOAD_TIMEOUT=120000 +# MAX_RETRIES=3 +``` + +### Generating secrets + +```bash +# Generate a strong JWT secret +openssl rand -hex 32 + +# Generate a strong password +openssl rand -base64 24 +``` + +--- + +## Building and Starting + +### Build the image + +```bash +docker-compose build +``` + +Only needed when `app.py`, `index.html`, or `requirements.txt` change. Skipped automatically if you just change `.env`. + +### Start the stack + +```bash +docker-compose up -d +``` + +The `-d` flag runs containers in the background (detached mode). Without it the logs stream to your terminal and the stack stops when you close the terminal. + +### Check that everything is running + +```bash +docker-compose ps +``` + +You should see all three containers with status `running` or `healthy`. + +### View logs + +```bash +# All containers +docker-compose logs -f + +# Just the web app +docker-compose logs -f rv50x-manager + +# Just NocoDB +docker-compose logs -f rv50x-nocodb +``` + +### Stop the stack + +```bash +docker-compose stop +``` + +Stops all containers. Data is preserved in volumes. Start again with `docker-compose up -d`. + +### Remove containers (keep data) + +```bash +docker-compose down +``` + +Removes containers but keeps all named volumes (your modem data, templates, downloads). Safe to run before a rebuild. + +### Remove everything including data ⚠ + +```bash +docker-compose down -v +``` + +**This deletes all volumes** including your NocoDB database and all template files. Only use this if you want a completely clean slate. + +--- + +## NocoDB Setup + +After starting the stack for the first time, NocoDB needs to be configured before the web app can use it. + +### Step 1 — Open NocoDB + +``` +http://your-docker-host-ip:8090 +``` + +### Step 2 — Create your account + +On first launch NocoDB will prompt you to create an admin account. Use a strong password and note the credentials. + +### Step 3 — Create a new base + +Click **+ New Base** and name it something like `Cell Modems`. + +### Step 4 — Import your modem data from CSV + +1. Export your current modem data from your existing NocoDB instance: **toolbar → Download → CSV** +2. In the new NocoDB, click **+ Add or import** → **Import from CSV** +3. Upload the CSV file +4. NocoDB will auto-detect all columns including `hostname`, `ip_address`, `dept`, `password`, etc. +5. Confirm the import + +### Step 5 — Recreate filtered views + +The CSV import brings the data but not the views. Recreate them manually: + +**Electric view:** +1. In the left sidebar, click **+ Add View** → **Grid** +2. Name it `Electric` +3. Click **Filter** → **+ Add Filter** +4. Set: `dept` `is` `ELEC` + +**Gas & Water view:** +1. Click **+ Add View** → **Grid** +2. Name it `Gas & Water` +3. Click **Filter** → **+ Add Filter** +4. Set: `dept` `is` `GW` + +### Step 6 — Get the IDs from the browser URL + +Navigate to your Cell Modems table. The URL will look like: + +``` +http://host:8090/abc123def/BASE_ID_HERE/TABLE_ID_HERE/VIEW_ID_HERE/cell-modems +``` + +Copy `BASE_ID_HERE`, `TABLE_ID_HERE`, and `VIEW_ID_HERE` (use the All view ID). + +### Step 7 — Generate an API token + +1. Click your profile avatar (bottom-left) +2. Go to **Team & Settings → API Tokens** +3. Click **Add Token**, give it a name like `rv50x-manager` +4. Copy the token + +### Step 8 — Update .env and restart + +```bash +cd /opt/rv50x-manager +nano .env +# Update NOCODB_TOKEN, NOCODB_BASE_ID, NOCODB_TABLE_ID, NOCODB_VIEW_ID + +docker-compose restart rv50x-manager +``` + +### Step 9 — Verify the connection + +Open the web app and check that the device groups show the correct counts. If devices appear, the connection is working. + +--- + +## Updating the App + +When `app.py` or `index.html` change: + +```bash +# Stop the stack +docker-compose stop + +# Rebuild the manager image (NocoDB and Postgres don't need rebuilding) +docker-compose build rv50x-manager + +# Start everything back up +docker-compose up -d +``` + +When only `.env` changes (NocoDB IDs, token, timeouts): + +```bash +docker-compose restart rv50x-manager +``` + +When `requirements.txt` changes (new Python packages): + +```bash +docker-compose build rv50x-manager +docker-compose up -d +``` + +--- + +## Managing the Stack + +### Start individual services + +```bash +docker-compose up -d postgres # start just postgres +docker-compose up -d nocodb # start just nocodb +docker-compose up -d rv50x-manager # start just the web app +``` + +### Restart a single service + +```bash +docker-compose restart rv50x-manager +``` + +### Rebuild and restart a single service + +```bash +docker-compose up -d --build rv50x-manager +``` + +### Execute a command inside a running container + +```bash +# Open a shell in the web app container +docker exec -it rv50x-manager bash + +# Check Python packages installed +docker exec rv50x-manager pip list + +# Test NocoDB connection from inside the container +docker exec rv50x-manager curl -s http://nocodb:8080/api/v1/health +``` + +### Access the PostgreSQL database directly + +```bash +docker exec -it rv50x-postgres psql -U nocodb -d nocodb +``` + +--- + +## Using Portainer + +Portainer gives you a browser-based UI to manage the entire stack without needing SSH. + +### Install Portainer (if not already installed) + +```bash +docker volume create portainer_data +docker run -d \ + -p 9000:9000 \ + --name portainer \ + --restart=always \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v portainer_data:/data \ + portainer/portainer-ce:latest +``` + +Then open `http://host-ip:9000` and create your admin account. + +### Deploy the stack via Portainer + +1. Go to **Stacks → + Add Stack** +2. Name it `rv50x` +3. Paste the contents of `docker-compose.yml` into the editor +4. Scroll down to **Environment Variables** +5. Click **+ Add an environment variable** for each line in your `.env` file: + +| Name | Value | +|---|---| +| `NOCODB_URL` | `http://nocodb:8080` | +| `NOCODB_TOKEN` | your token | +| `NOCODB_BASE_ID` | your base ID | +| `NOCODB_TABLE_ID` | your table ID | +| `NOCODB_VIEW_ID` | your view ID | +| `POSTGRES_PASSWORD` | your password | +| `NC_JWT_SECRET` | your secret | + +6. Click **Deploy the stack** + +### Start and stop via Portainer + +- **Stacks → rv50x → Start** — starts all containers +- **Stacks → rv50x → Stop** — stops all containers +- Individual containers: **Containers** list → click the start/stop icons + +### Update the stack via Portainer + +1. **Stacks → rv50x → Editor** +2. Paste updated `docker-compose.yml` +3. Click **Update the stack** + +For code changes (`app.py`, `index.html`), you need to rebuild the image first from the command line: + +```bash +docker-compose build rv50x-manager +``` + +Then update the stack in Portainer to pick up the new image. + +--- + +## Migrating to a New Machine + +### Step 1 — Export your NocoDB data + +Before migrating, export your modem data as CSV from the current NocoDB instance so you can reimport it on the new machine. + +In NocoDB: **Cell Modems table → toolbar → Download → CSV** + +### Step 2 — Copy project files + +From the old machine: + +```bash +scp app.py index.html requirements.txt Dockerfile docker-compose.yml \ + .env.example .dockerignore youruser@new-host:/tmp/rv50x-transfer/ +``` + +**Do not copy `.env`** over an insecure connection — recreate it manually on the new machine. + +On the new machine, move files into place: + +```bash +sudo mkdir -p /opt/rv50x-manager +sudo chown $USER:$USER /opt/rv50x-manager +cp /tmp/rv50x-transfer/* /opt/rv50x-manager/ +``` + +### Step 3 — Copy template files (optional) + +If you want to keep your existing XML templates, downloaded configs, and upload files: + +```bash +# Export volumes from the old machine +cd /opt/rv50x-manager + +docker run --rm \ + -v rv50x-manager_xml_templates:/data \ + -v $(pwd):/backup \ + alpine tar czf /backup/xml_templates.tar.gz -C /data . + +docker run --rm \ + -v rv50x-manager_template_downloads:/data \ + -v $(pwd):/backup \ + alpine tar czf /backup/template_downloads.tar.gz -C /data . + +docker run --rm \ + -v rv50x-manager_template_uploads:/data \ + -v $(pwd):/backup \ + alpine tar czf /backup/template_uploads.tar.gz -C /data . + +# Copy archives to new machine +scp xml_templates.tar.gz template_downloads.tar.gz template_uploads.tar.gz \ + youruser@new-host:/opt/rv50x-manager/ +``` + +### Step 4 — Set up the new machine + +```bash +# Install Docker (see First-Time Setup above) + +# Files should already be in /opt/rv50x-manager from Step 2 +cd /opt/rv50x-manager + +# Create fresh .env with new passwords and secrets +cp .env.example .env +nano .env + +# Set your port +nano docker-compose.yml + +# Build and start +docker-compose up -d +``` + +### Step 5 — Restore template files (if copied) + +```bash +cd /opt/rv50x-manager + +# Restore xml_templates +docker run --rm \ + -v rv50x-manager_xml_templates:/data \ + -v $(pwd):/backup \ + alpine tar xzf /backup/xml_templates.tar.gz -C /data + +# Restore template_downloads +docker run --rm \ + -v rv50x-manager_template_downloads:/data \ + -v $(pwd):/backup \ + alpine tar xzf /backup/template_downloads.tar.gz -C /data + +# Restore template_uploads +docker run --rm \ + -v rv50x-manager_template_uploads:/data \ + -v $(pwd):/backup \ + alpine tar xzf /backup/template_uploads.tar.gz -C /data +``` + +### Step 6 — Set up NocoDB on the new machine + +Follow the [NocoDB Setup](#nocodb-setup) section — import your CSV, recreate views, get new IDs. + +### Step 7 — Update .env with new NocoDB IDs + +```bash +nano .env +docker-compose restart rv50x-manager +``` + +--- + +## Reconnecting to a New NocoDB Instance + +If your NocoDB database is corrupted, rebuilt, or moved to a new server: + +### Step 1 — Get the new connection details + +1. Open the new NocoDB in your browser +2. Import your modem data from CSV +3. Recreate the Electric and Gas & Water filtered views +4. Go to **Profile → Team & Settings → API Tokens** → create a new token +5. Copy the base ID, table ID, and view ID from the browser URL + +### Step 2 — Test the connection + +```bash +curl -H "xc-token: YOUR_NEW_TOKEN" \ + "http://new-nocodb-host:8090/api/v1/db/data/noco/BASE_ID/TABLE_ID?limit=1" +``` + +A successful response returns JSON with a `list` array containing your first modem row. + +### Step 3 — Update .env + +```bash +cd /opt/rv50x-manager +nano .env +``` + +Update these values: + +```bash +NOCODB_URL=http://new-host:8090 # or http://nocodb:8080 if using the stack +NOCODB_TOKEN=your-new-token +NOCODB_BASE_ID=your-new-base-id +NOCODB_TABLE_ID=your-new-table-id +NOCODB_VIEW_ID=your-new-view-id +``` + +### Step 4 — Restart the manager + +```bash +cd /opt/rv50x-manager +docker-compose restart rv50x-manager +``` + +Or in Portainer: **Stacks → rv50x → rv50x-manager → Restart** + +--- + +## Data and Volumes + +All persistent data lives in Docker named volumes. They survive `docker-compose down` and rebuilds, and are only deleted with `docker-compose down -v`. + +| Volume | Contents | Maps to container path | +|---|---|---| +| `rv50x_template_manager_postgres_data` | NocoDB database | `/var/lib/postgresql/data` | +| `rv50x_template_manager_nocodb_data` | NocoDB config and uploads | `/usr/app/data` | +| `rv50x_template_manager_template_downloads` | Downloaded modem configs + reports | `/data/template_downloads` | +| `rv50x_template_manager_template_uploads` | Staged XML files for upload + reports | `/data/template_uploads` | +| `rv50x_template_manager_xml_templates` | Your XML builder templates | `/data/xml_templates` | + +### List all volumes + +```bash +docker volume ls | grep rv50x +``` + +### Back up a volume + +```bash +# Backs up the xml_templates volume to a tar file in the current directory +docker run --rm \ + -v rv50x_template_manager_xml_templates:/data \ + -v $(pwd):/backup \ + alpine tar czf /backup/xml_templates_backup.tar.gz -C /data . +``` + +### Back up the NocoDB database + +```bash +cd /opt/rv50x-manager +docker exec rv50x-postgres \ + pg_dump -U nocodb nocodb > nocodb_backup_$(date +%Y%m%d).sql +``` + +### Restore the NocoDB database + +```bash +cd /opt/rv50x-manager +docker exec -i rv50x-postgres \ + psql -U nocodb nocodb < nocodb_backup_20260413.sql +``` + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `docker-compose: command not found` | Using older Docker without compose plugin | Use `docker compose` (space not hyphen) or install `docker-compose-plugin` | +| Build fails with "no space left on device" | Docker image cache full | Run `docker system prune` to free space | +| `rv50x-manager` exits immediately after start | App crash on startup — bad .env values | Run `docker-compose logs rv50x-manager` to see the error | +| NocoDB shows "Service Unavailable" | Postgres not ready yet | Wait 30s and refresh — healthcheck retries handle this automatically | +| Web app shows 0 devices | NocoDB IDs wrong or token invalid | Test with curl (see Reconnecting section), update .env, restart manager | +| `ERR_BASE_NOT_FOUND` in curl test | Wrong NOCODB_BASE_ID | Re-read the ID from the NocoDB browser URL | +| `ERR_AUTHENTICATION_REQUIRED` | Wrong or expired token | Regenerate token in NocoDB → API Tokens | +| Modems show in All but not Electric/Gas & Water | Wrong dept field values | Check actual `dept` values in NocoDB — must be exactly `ELEC` and `GW` | +| Playwright / Chromium crashes | Missing system library | The Playwright base image should have everything — check `docker-compose logs rv50x-manager` | +| Template upload times out | Modem slow or unreachable | Increase `UPLOAD_TIMEOUT` in `.env`, restart manager | +| Can't reach NocoDB at port 8090 | Port conflict or firewall | Change the left port in `docker-compose.yml` under `nocodb:` ports | +| Can't reach web app | Port conflict or firewall | Change `YOUR_PORT` in `docker-compose.yml` | +| `permission denied` on docker commands | User not in docker group | Run `sudo usermod -aG docker $USER` then log out and back in | +| Volume data missing after `docker-compose down` | Used `down -v` by mistake | Data is gone — restore from backup or re-import CSV | diff --git a/app.py b/app.py new file mode 100644 index 0000000..0346642 --- /dev/null +++ b/app.py @@ -0,0 +1,1277 @@ +""" +RV50x Template Manager - Web GUI Backend + +Run directly: + uvicorn app:app --host 0.0.0.0 --port 8000 + +Run via Docker: + docker-compose up -d + +Environment variables (set in .env or docker-compose.yml): + NOCODB_URL NocoDB base URL e.g. http://192.168.16.130:8080 + NOCODB_TOKEN NocoDB API token + NOCODB_BASE_ID Base ID from browser URL + NOCODB_TABLE_ID Table ID from browser URL + NOCODB_VIEW_ID View ID from browser URL (the "All" view) + PAGE_TIMEOUT ms to wait for page elements (default 90000) + DOWNLOAD_TIMEOUT ms to wait for template gen (default 120000) + UPLOAD_TIMEOUT ms to wait for upload done (default 120000) + MAX_RETRIES retry attempts per device (default 3) +""" + +import asyncio +import html as html_lib +import io +import json +import os +import re +import threading +import urllib.parse +import urllib.request +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +import pandas as pd +import paramiko +from fastapi import FastAPI, BackgroundTasks, File, Form, UploadFile, Request, Response, Depends, HTTPException +from fastapi.responses import HTMLResponse, StreamingResponse, RedirectResponse +from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired +from pydantic import BaseModel + +# ── NocoDB connection — read from environment, fall back to defaults ─────────── +NOCODB_URL = os.environ.get("NOCODB_URL", "http://192.168.16.130:8080") +NOCODB_TOKEN = os.environ.get("NOCODB_TOKEN", "eWU_ilelaCtNy1JzC7vf41DokkqFOovcLHM0zVml") +NOCODB_BASE_ID = os.environ.get("NOCODB_BASE_ID", "pdq96x915xt4a0m") +NOCODB_TABLE_ID = os.environ.get("NOCODB_TABLE_ID", "mkewnr53ahqvnt9") +_view_id = os.environ.get("NOCODB_VIEW_ID", "vwl7qvxo1xclvawz") +NOCODB_VIEW_IDS = { + "All": _view_id, + "Electric": _view_id, + "Gas & Water": _view_id, +} + +# ── Local paths ──────────────────────────────────────────────────────────────── +# In Docker these are bind-mounted from the host so data persists across rebuilds. +SCRIPT_DIR = Path(__file__).parent +DOWNLOAD_DIR = Path(os.environ.get("DOWNLOAD_DIR", str(SCRIPT_DIR / "template_downloads"))) +UPLOAD_DIR = Path(os.environ.get("UPLOAD_DIR", str(SCRIPT_DIR / "template_uploads"))) +TEMPLATES_DIR = Path(os.environ.get("TEMPLATES_DIR", str(SCRIPT_DIR / "xml_templates"))) +DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True) +UPLOAD_DIR.mkdir(parents=True, exist_ok=True) +TEMPLATES_DIR.mkdir(parents=True, exist_ok=True) + +# ── Playwright timeouts — tunable via environment ────────────────────────────── +PAGE_TIMEOUT = int(os.environ.get("PAGE_TIMEOUT", "90000")) +DOWNLOAD_TIMEOUT = int(os.environ.get("DOWNLOAD_TIMEOUT", "120000")) +UPLOAD_TIMEOUT = int(os.environ.get("UPLOAD_TIMEOUT", "120000")) +MAX_RETRIES = int(os.environ.get("MAX_RETRIES", "3")) + +# ── In-memory job store ──────────────────────────────────────────────────────── +jobs: dict[str, dict] = {} +abort_flags: set[str] = set() # job_ids that have been asked to abort + +# ── AT Terminal job store (separate from Playwright jobs) ────────────────────── +at_sessions: dict[str, dict] = {} # session_id → {logs, status, results} +SSH_PORT = int(os.environ.get("SSH_PORT", "3223")) + +# ── Auth config ──────────────────────────────────────────────────────────────── +APP_USERNAME = os.environ.get("APP_USERNAME", "admin") +APP_PASSWORD = os.environ.get("APP_PASSWORD", "changeme") +SESSION_SECRET = os.environ.get("SESSION_SECRET", "change-this-secret") +SESSION_HOURS = int(os.environ.get("SESSION_HOURS", "8")) +SESSION_COOKIE = "rv50x_session" +_signer = URLSafeTimedSerializer(SESSION_SECRET) + +def _make_session_cookie() -> str: + return _signer.dumps({"user": APP_USERNAME}) + +def _verify_session_cookie(token: str) -> bool: + try: + _signer.loads(token, max_age=SESSION_HOURS * 3600) + return True + except (BadSignature, SignatureExpired): + return False + +def require_auth(request: Request): + token = request.cookies.get(SESSION_COOKIE) + if not token or not _verify_session_cookie(token): + raise HTTPException(status_code=302, headers={"Location": "/login"}) + +app = FastAPI(title="RV50x Template Manager") + + +# ══════════════════════════════════════════════════════════════════════════════ +# NocoDB helpers +# ══════════════════════════════════════════════════════════════════════════════ + +def _fetch_all_rows(view_id: str) -> list: + headers = {"xc-token": NOCODB_TOKEN, "Content-Type": "application/json"} + all_rows = [] + offset, limit = 0, 1000 + while True: + params = urllib.parse.urlencode({"viewId": view_id, "limit": limit, "offset": offset}) + url = f"{NOCODB_URL}/api/v1/db/data/noco/{NOCODB_BASE_ID}/{NOCODB_TABLE_ID}?{params}" + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.loads(resp.read().decode()) + all_rows.extend(data.get("list", [])) + if data.get("pageInfo", {}).get("isLastPage", True): + break + offset += limit + return all_rows + + +# All NocoDB fields we care about — everything becomes a template variable +NOCODB_FIELDS = [ + "hostname", "ip_address", "username", "password", "https_port", + "use_https", "ssh_port", "location", "alias", "dept", "def_pass", + "snmp_auth_key", "snmp_priv_key", "fort_key", "vpn_subnets", +] + + +def load_devices(group: str = "All") -> list: + view_id = NOCODB_VIEW_IDS.get(group, NOCODB_VIEW_IDS["All"]) + rows = _fetch_all_rows(view_id) + devices = [] + for row in rows: + device_id = str(row.get("hostname", "")).strip() + ip = str(row.get("ip_address", "")).strip() + username = str(row.get("username", "")).strip() + password = str(row.get("password", "")).strip() + dept = str(row.get("dept", "")).strip() + location = str(row.get("location", "")).strip() + if not device_id or not ip: + continue + if group == "Electric" and dept.upper() != "ELEC": + continue + if group == "Gas & Water" and dept.upper() != "GW": + continue + devices.append({ + "id": device_id, "ip": ip, + "username": username, "password": password, + "dept": dept, "location": location, + }) + return devices + + +def load_devices_full(group: str = "All") -> list: + """Load all NocoDB fields for each device (used by XML Builder).""" + view_id = NOCODB_VIEW_IDS.get(group, NOCODB_VIEW_IDS["All"]) + rows = _fetch_all_rows(view_id) + devices = [] + for row in rows: + device_id = str(row.get("hostname", "")).strip() + dept = str(row.get("dept", "")).strip() + if not device_id: + continue + if group == "Electric" and dept.upper() != "ELEC": + continue + if group == "Gas & Water" and dept.upper() != "GW": + continue + # Build a flat dict of all available fields + device = {} + for field in NOCODB_FIELDS: + val = row.get(field, "") + device[field] = str(val).strip() if val is not None else "" + devices.append(device) + return devices + + +# ══════════════════════════════════════════════════════════════════════════════ +# XML Builder — core logic (replaces excel_to_xml.py) +# ══════════════════════════════════════════════════════════════════════════════ + +def _extract_template_variables(template: str) -> list[str]: + """Return unique variable names found in {var} or {{ var }} placeholders.""" + pattern = r'\{\{?\s*([^{}]+?)\s*\}?\}' + return list(set(re.findall(pattern, template))) + + +def _escape_xml(value) -> str: + """Convert a value to an XML-safe string.""" + if not isinstance(value, str): + # Convert float-integers like 443.0 → "443" + if isinstance(value, float) and value == int(value): + value = str(int(value)) + else: + value = str(value) + # Escape backslashes then XML special chars + value = value.replace('\\', '\\\\') + value = html_lib.escape(value) + return value + + +def _render_template(template: str, data: dict, default: str = "NOTSET") -> str: + """Replace all {var} and {{ var }} placeholders in template with data values.""" + for var, raw_val in data.items(): + val = _escape_xml(raw_val) if raw_val not in ("", None) else default + try: + template = re.sub(r'\{\{\s*' + re.escape(var) + r'\s*\}\}', val, template) + template = re.sub(r'\{' + re.escape(var) + r'\}', val, template) + except Exception: + pass + # Fill any remaining unreplaced variables with the default + template = re.sub(r'\{\{?\s*[^{}]+?\s*\}?\}', default, template) + return template + + +def build_xmls_from_nocodb( + group: str, + hostnames: list[str], + template: str, + default: str = "NOTSET", +) -> list[dict]: + """ + Generate XML files for the given hostnames using live NocoDB data. + Files are written to template_uploads/. + Returns a list of result dicts: {hostname, success, message}. + """ + all_devices = load_devices_full(group) + device_map = {d["hostname"]: d for d in all_devices} + results = [] + + for hostname in hostnames: + if hostname not in device_map: + results.append({"hostname": hostname, "success": False, "message": "Not found in NocoDB"}) + continue + data = device_map[hostname] + content = _render_template(template, data, default) + out = UPLOAD_DIR / f"{hostname}.xml" + try: + out.write_text(content, encoding="utf-8") + results.append({"hostname": hostname, "success": True, "message": f"→ {out.name}"}) + except Exception as e: + results.append({"hostname": hostname, "success": False, "message": str(e)}) + + return results + + +def build_xmls_from_excel( + excel_bytes: bytes, + template: str, + hostnames: list[str], + default: str = "NOTSET", +) -> list[dict]: + """ + Generate XML files for the given hostnames using data from an uploaded Excel file. + Files are written to template_uploads/. + Returns a list of result dicts: {hostname, success, message}. + """ + df = pd.read_excel(io.BytesIO(excel_bytes)) + + # Find hostname column + hostname_col = next( + (c for c in df.columns if "hostname" in c.lower()), None + ) + if not hostname_col: + return [{"hostname": h, "success": False, "message": "No hostname column in Excel"} for h in hostnames] + + results = [] + for hostname in hostnames: + rows = df[df[hostname_col] == hostname] + if rows.empty: + results.append({"hostname": hostname, "success": False, "message": "Not found in Excel"}) + continue + + row = rows.iloc[0].to_dict() + data = {} + for col, val in row.items(): + clean = col.strip().strip('{}') + if pd.notna(val): + if isinstance(val, float) and val == int(val): + data[clean] = str(int(val)) + else: + data[clean] = str(val) + else: + data[clean] = default + + content = _render_template(template, data, default) + out = UPLOAD_DIR / f"{hostname}.xml" + try: + out.write_text(content, encoding="utf-8") + results.append({"hostname": hostname, "success": True, "message": f"→ {out.name}"}) + except Exception as e: + results.append({"hostname": hostname, "success": False, "message": str(e)}) + + return results + + +# ══════════════════════════════════════════════════════════════════════════════ +# Report writer +# ══════════════════════════════════════════════════════════════════════════════ + +def _write_report(job_id: str, job: dict, output_dir: Path) -> Path: + """Write a timestamped report file matching the CLI format.""" + results = job.get("results", []) + succeeded = [r for r in results if r.get("success")] + failed = [r for r in results if not r.get("success")] + started = datetime.fromisoformat(job["started"]) if job.get("started") else datetime.now() + finished = datetime.fromisoformat(job["finished"]) if job.get("finished") else datetime.now() + elapsed = (finished - started).total_seconds() + job_type = job.get("type", "job").upper() + timestamp = finished.strftime("%Y-%m-%d %H:%M") + + lines = [] + lines.append("═" * 60) + lines.append(f" {job_type} SUMMARY — {timestamp}") + lines.append(f" Job ID: {job_id}") + lines.append("═" * 60) + lines.append(f" Total: {len(results)}") + lines.append(f" ✓ Success: {len(succeeded)}") + lines.append(f" ✗ Failed: {len(failed)}") + lines.append(f" Time: {elapsed:.0f}s") + if succeeded: + lines.append("") + lines.append(f" {'Downloaded' if job_type == 'DOWNLOAD' else 'Uploaded'}:") + for r in succeeded: + extra = f" ({r.get('size_kb', '')} KB)" if r.get("size_kb") else "" + lines.append(f" ✓ {r['id']}{extra}") + if failed: + lines.append("") + lines.append(" Failed:") + for r in failed: + lines.append(f" ✗ {r['id']} — {r.get('message', '')}") + lines.append("═" * 60) + lines.append("") + lines.append(" Full log:") + lines.append("") + for log_line in job.get("logs", []): + lines.append(f" {log_line}") + lines.append("═" * 60) + + report_name = f"report_{finished.strftime('%Y%m%d_%H%M%S')}.txt" + report_path = output_dir / report_name + report_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return report_path + + +# ══════════════════════════════════════════════════════════════════════════════ +# Playwright workers (download / upload to modems) +# ══════════════════════════════════════════════════════════════════════════════ + +async def _download_one(device: dict, job_id: str) -> dict: + from playwright.async_api import async_playwright, TimeoutError as PWTimeout + device_id = device["id"] + ip = device["ip"] + username = device["username"] + password = device["password"] + + def log(msg): + jobs[job_id]["logs"].append(f"[{datetime.now().strftime('%H:%M:%S')}] [{device_id}] {msg}") + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context(ignore_https_errors=True, accept_downloads=True) + page = await context.new_page() + page.set_default_timeout(PAGE_TIMEOUT) + try: + await page.goto(f"https://{ip}", wait_until="load") + await page.wait_for_selector('input[name="username"]', state="visible", timeout=PAGE_TIMEOUT) + await page.wait_for_timeout(1000) + await page.click('input[name="username"]', click_count=3) + await page.fill('input[name="username"]', username) + await page.fill('input[name="password"]', password) + await page.wait_for_timeout(800) + login_btn = page.locator('input[name="Login"]') + await login_btn.scroll_into_view_if_needed() + await login_btn.click() + await page.wait_for_selector('#btn_tpl', state="visible", timeout=PAGE_TIMEOUT) + log("Logged in.") + await page.wait_for_function("typeof showTemplateDialog === 'function'", timeout=PAGE_TIMEOUT) + await page.wait_for_timeout(500) + + for _ in range(5): + await page.evaluate("showTemplateDialog()") + await page.wait_for_timeout(1000) + visible = await page.evaluate(""" + () => { + const el = document.getElementById('template_name'); + if (!el) return false; + const form = el.closest('form') || el.closest('div[id]'); + if (!form) return el.offsetParent !== null; + return form.offsetParent !== null; + } + """) + if visible: + break + else: + return {"id": device_id, "success": False, "message": "Modal did not open"} + + await page.fill('input[id="template_name"]', device_id) + pwd_cb = page.locator('input[id="template_pwds"]') + if not await pwd_cb.is_checked(): + await pwd_cb.check() + info_cb = page.locator('input[id="template_info"]') + if await info_cb.count() > 0 and await info_cb.is_checked(): + await info_cb.uncheck() + + page.on("dialog", lambda d: asyncio.ensure_future(d.dismiss())) + out_path = DOWNLOAD_DIR / f"{device_id}.xml" + async with page.expect_download(timeout=DOWNLOAD_TIMEOUT) as dl: + await page.click('input.fbtn[name="download_template"]') + download = await dl.value + await download.save_as(out_path) + await page.wait_for_timeout(1000) + + if out_path.exists() and out_path.stat().st_size > 0: + size_kb = out_path.stat().st_size // 1024 + log(f"✓ Saved {size_kb} KB") + return {"id": device_id, "success": True, "message": f"{size_kb} KB", "size_kb": size_kb} + return {"id": device_id, "success": False, "message": "File empty after download"} + + except PWTimeout: + log("✗ TIMEOUT") + return {"id": device_id, "success": False, "message": "TIMEOUT"} + except Exception as e: + log(f"✗ ERROR: {str(e)[:80]}") + return {"id": device_id, "success": False, "message": str(e)[:120]} + finally: + await context.close() + await browser.close() + + +async def _upload_one(device: dict, reboot: bool, job_id: str) -> dict: + from playwright.async_api import async_playwright, TimeoutError as PWTimeout + device_id = device["id"] + ip = device["ip"] + username = device["username"] + password = device["password"] + template_file = UPLOAD_DIR / f"{device_id}.xml" + + def log(msg): + jobs[job_id]["logs"].append(f"[{datetime.now().strftime('%H:%M:%S')}] [{device_id}] {msg}") + + if not template_file.exists(): + log("✗ Template file not found") + return {"id": device_id, "success": False, "message": "Template file not found"} + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context(ignore_https_errors=True, accept_downloads=True) + page = await context.new_page() + page.set_default_timeout(PAGE_TIMEOUT) + try: + await page.goto(f"https://{ip}", wait_until="load") + await page.wait_for_selector('input[name="username"]', state="visible", timeout=PAGE_TIMEOUT) + await page.wait_for_timeout(1000) + await page.click('input[name="username"]', click_count=3) + await page.fill('input[name="username"]', username) + await page.fill('input[name="password"]', password) + await page.wait_for_timeout(800) + login_btn = page.locator('input[name="Login"]') + await login_btn.scroll_into_view_if_needed() + await login_btn.click() + await page.wait_for_selector('#btn_tpl', state="visible", timeout=PAGE_TIMEOUT) + await page.wait_for_function("typeof showTemplateDialog === 'function'", timeout=PAGE_TIMEOUT) + await page.wait_for_timeout(500) + log("Logged in.") + + for _ in range(5): + await page.evaluate("showTemplateDialog()") + await page.wait_for_timeout(1000) + visible = await page.evaluate(""" + () => { + const el = document.getElementById('template_name'); + if (!el) return false; + const form = el.closest('form') || el.closest('div[id]'); + if (!form) return el.offsetParent !== null; + return form.offsetParent !== null; + } + """) + if visible: + break + else: + return {"id": device_id, "success": False, "message": "Modal did not open"} + + await page.locator('input[id="template_filename"]').set_input_files(str(template_file)) + log(f"File set: {template_file.name}") + + reboot_cb = page.locator('input[name="reboot_after_upload"]') + is_checked = await reboot_cb.is_checked() + if reboot and not is_checked: + await reboot_cb.check() + elif not reboot and is_checked: + await reboot_cb.uncheck() + + page.on("dialog", lambda d: asyncio.ensure_future(d.dismiss())) + log("Uploading...") + + if reboot: + # Click upload then race multiple reboot signals. + # Different firmware versions signal reboot differently: + # - URL navigates to /?reboot + # - Page shows a reboot/success message + # - Connection drops entirely (device gone) + await page.click('input[name="upload_template"]') + + reboot_confirmed = False + deadline = UPLOAD_TIMEOUT / 1000 # seconds + + async def _any_reboot_signal(): + tasks = [ + asyncio.ensure_future(page.wait_for_url("**/?reboot**", timeout=UPLOAD_TIMEOUT)), + asyncio.ensure_future(page.wait_for_url("**reboot**", timeout=UPLOAD_TIMEOUT)), + asyncio.ensure_future(page.wait_for_selector( + 'text=Rebooting, text=reboot, text=Upload Complete, text=Apply Complete', + timeout=UPLOAD_TIMEOUT + )), + ] + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for t in pending: + t.cancel() + return bool(done) + + try: + reboot_confirmed = await asyncio.wait_for(_any_reboot_signal(), timeout=deadline) + except (asyncio.TimeoutError, Exception): + # Connection drop or page crash after clicking upload + # is itself strong evidence the device is rebooting. + try: + await page.title() + except Exception: + reboot_confirmed = True # page is gone = rebooted + + if reboot_confirmed: + log("✓ Rebooting — upload successful.") + return {"id": device_id, "success": True, "message": "OK"} + + log("✗ Reboot signal not detected after upload") + return {"id": device_id, "success": False, "message": "Reboot not confirmed"} + + else: + await page.click('input[name="upload_template"]') + await page.wait_for_selector( + 'text=Upload Complete, text=Apply Complete, text=Template Applied', + timeout=UPLOAD_TIMEOUT + ) + log("✓ Upload complete (no reboot).") + return {"id": device_id, "success": True, "message": "OK"} + + except PWTimeout: + log("✗ TIMEOUT") + return {"id": device_id, "success": False, "message": "TIMEOUT"} + except Exception as e: + err = str(e).lower() + # Connection errors AFTER upload started = device rebooted + if reboot and any(k in err for k in ("connection", "net::", "closed", "crashed", "target", "disconnected")): + log("✓ Connection closed — device is rebooting.") + return {"id": device_id, "success": True, "message": "OK"} + log(f"✗ ERROR: {str(e)[:80]}") + return {"id": device_id, "success": False, "message": str(e)[:120]} + finally: + await context.close() + await browser.close() + + +async def _run_with_retry(fn, device, job_id, **kwargs): + device_id = device["id"] + ts = lambda: datetime.now().strftime('%H:%M:%S') + + for attempt in range(1, MAX_RETRIES + 1): + # Check abort flag before each attempt + if job_id in abort_flags: + jobs[job_id]["logs"].append(f"[{ts()}] [{device_id}] ✗ Aborted.") + return {"id": device_id, "success": False, "message": "Aborted"} + + jobs[job_id]["logs"].append( + f"[{ts()}] [{device_id}] Attempt {attempt}/{MAX_RETRIES}" + ) + result = await fn(device, job_id=job_id, **kwargs) + + if result["success"]: + return result + + if attempt < MAX_RETRIES: + if job_id in abort_flags: + jobs[job_id]["logs"].append(f"[{ts()}] [{device_id}] ✗ Aborted.") + return {"id": device_id, "success": False, "message": "Aborted"} + jobs[job_id]["logs"].append( + f"[{ts()}] [{device_id}] Retrying in 15s..." + ) + # Poll abort flag every second so abort is responsive + for _ in range(15): + if job_id in abort_flags: + break + await asyncio.sleep(1) + + return result + + +# ══════════════════════════════════════════════════════════════════════════════ +# Background job runners +# ══════════════════════════════════════════════════════════════════════════════ + +async def run_download_job(job_id: str, devices: list, concurrency: int): + jobs[job_id]["status"] = "running" + semaphore = asyncio.Semaphore(concurrency) + + async def _guarded(device): + async with semaphore: + return await _run_with_retry(_download_one, device, job_id) + + results = await asyncio.gather(*[_guarded(d) for d in devices]) + jobs[job_id]["results"] = list(results) + jobs[job_id]["status"] = "done" + jobs[job_id]["finished"] = datetime.now().isoformat() + abort_flags.discard(job_id) + report_path = _write_report(job_id, jobs[job_id], DOWNLOAD_DIR) + jobs[job_id]["logs"].append( + f"[{datetime.now().strftime('%H:%M:%S')}] Report saved → {report_path.name}" + ) + + +async def run_upload_job(job_id: str, devices: list, concurrency: int, reboot: bool): + jobs[job_id]["status"] = "running" + semaphore = asyncio.Semaphore(concurrency) + + async def _guarded(device): + async with semaphore: + return await _run_with_retry(_upload_one, device, job_id, reboot=reboot) + + results = await asyncio.gather(*[_guarded(d) for d in devices]) + jobs[job_id]["results"] = list(results) + jobs[job_id]["status"] = "done" + jobs[job_id]["finished"] = datetime.now().isoformat() + abort_flags.discard(job_id) + report_path = _write_report(job_id, jobs[job_id], UPLOAD_DIR) + jobs[job_id]["logs"].append( + f"[{datetime.now().strftime('%H:%M:%S')}] Report saved → {report_path.name}" + ) + + +# ══════════════════════════════════════════════════════════════════════════════ +# API models +# ══════════════════════════════════════════════════════════════════════════════ + +class JobRequest(BaseModel): + group: str = "All" + device_ids: list[str] = [] + concurrency: int = 1 + reboot: bool = True + + +class BuildRequest(BaseModel): + source: str # "nocodb" or "excel" + group: str = "All" + hostnames: list[str] + template_name: str # filename in xml_templates/ + default_value: str = "NOTSET" + + +# ══════════════════════════════════════════════════════════════════════════════ +# API routes — devices & jobs (unchanged) +# ══════════════════════════════════════════════════════════════════════════════ + +@app.get("/api/devices") +def api_devices(request: Request, group: str = "All", _auth=Depends(require_auth)): + try: + devices = load_devices(group) + return {"devices": devices, "count": len(devices)} + except Exception as e: + return {"error": str(e), "devices": [], "count": 0} + + +@app.post("/api/jobs/download") +async def start_download(request: Request, req: JobRequest, background_tasks: BackgroundTasks, _auth=Depends(require_auth)): + devices = load_devices(req.group) + if req.device_ids: + devices = [d for d in devices if d["id"] in req.device_ids] + if not devices: + return {"error": "No matching devices found"} + job_id = f"dl_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + jobs[job_id] = { + "type": "download", "status": "queued", + "logs": [], "results": [], + "started": datetime.now().isoformat(), "finished": None, + "device_count": len(devices), + } + background_tasks.add_task(run_download_job, job_id, devices, req.concurrency) + return {"job_id": job_id} + + +@app.post("/api/jobs/upload") +async def start_upload(request: Request, req: JobRequest, background_tasks: BackgroundTasks, _auth=Depends(require_auth)): + devices = load_devices(req.group) + if req.device_ids: + devices = [d for d in devices if d["id"] in req.device_ids] + devices = [d for d in devices if (UPLOAD_DIR / f"{d['id']}.xml").exists()] + if not devices: + return {"error": "No devices with matching template files found in template_uploads/"} + job_id = f"ul_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + jobs[job_id] = { + "type": "upload", "status": "queued", + "logs": [], "results": [], + "started": datetime.now().isoformat(), "finished": None, + "device_count": len(devices), + "reboot": req.reboot, + } + background_tasks.add_task(run_upload_job, job_id, devices, req.concurrency, req.reboot) + return {"job_id": job_id} + + +@app.get("/api/jobs/{job_id}") +def get_job(request: Request, job_id: str, _auth=Depends(require_auth)): + if job_id not in jobs: + return {"error": "Job not found"} + return jobs[job_id] + + +@app.get("/api/jobs/{job_id}/stream") +async def stream_job(request: Request, job_id: str, from_line: int = 0, _auth=Depends(require_auth)): + """Stream job logs via SSE. Pass ?from_line=N to skip already-seen lines.""" + async def event_generator(): + sent = max(0, from_line) + while True: + if job_id not in jobs: + yield "data: {\"error\": \"job not found\"}\n\n" + break + job = jobs[job_id] + logs = job["logs"] + if len(logs) > sent: + for line in logs[sent:]: + yield f"data: {json.dumps({'log': line, 'line_num': sent})}\n\n" + sent += 1 + if job["status"] == "done": + yield f"data: {json.dumps({'done': True, 'results': job['results']})}\n\n" + break + await asyncio.sleep(0.5) + return StreamingResponse(event_generator(), media_type="text/event-stream") + + +@app.get("/api/jobs") +def list_jobs(request: Request, _auth=Depends(require_auth)): + summary = [] + for jid, j in sorted(jobs.items(), reverse=True): + summary.append({ + "job_id": jid, "type": j["type"], "status": j["status"], + "device_count": j["device_count"], "started": j["started"], + "finished": j.get("finished"), + "succeeded": sum(1 for r in j["results"] if r.get("success")), + "failed": sum(1 for r in j["results"] if not r.get("success")), + }) + return {"jobs": summary} + + +@app.get("/api/files/downloads") +def list_downloads(request: Request, _auth=Depends(require_auth)): + files = sorted(DOWNLOAD_DIR.glob("*.xml")) + return {"files": [{"name": f.name, "size_kb": f.stat().st_size // 1024} for f in files]} + + +@app.get("/api/files/uploads") +def list_uploads(request: Request, _auth=Depends(require_auth)): + files = sorted(UPLOAD_DIR.glob("*.xml")) + return {"files": [{"name": f.name, "size_kb": f.stat().st_size // 1024} for f in files]} + + +# ══════════════════════════════════════════════════════════════════════════════ +# API routes — XML Builder +# ══════════════════════════════════════════════════════════════════════════════ + +@app.get("/api/xmlbuilder/nocodb-fields") +def xmlbuilder_nocodb_fields(request: Request, _auth=Depends(require_auth)): + """Return available NocoDB field names (= available template variables).""" + return {"fields": NOCODB_FIELDS} + + +@app.get("/api/xmlbuilder/nocodb-devices") +def xmlbuilder_nocodb_devices(request: Request, group: str = "All", _auth=Depends(require_auth)): + """Return hostnames for the given group from NocoDB.""" + try: + devices = load_devices_full(group) + hostnames = [d["hostname"] for d in devices if d["hostname"]] + return {"hostnames": hostnames, "count": len(hostnames)} + except Exception as e: + return {"error": str(e), "hostnames": []} + + +@app.post("/api/xmlbuilder/upload-excel") +async def xmlbuilder_upload_excel(request: Request, file: UploadFile = File(...), _auth=Depends(require_auth)): + """ + Accept an uploaded Excel file, return its columns and hostnames. + Stores the file temporarily in memory (not on disk). + """ + try: + contents = await file.read() + df = pd.read_excel(io.BytesIO(contents)) + columns = [c.strip().strip('{}') for c in df.columns] + hostname_col = next((c for c in df.columns if "hostname" in c.lower()), None) + hostnames = df[hostname_col].dropna().astype(str).tolist() if hostname_col else [] + # Store bytes in a simple cache keyed by filename + excel_cache[file.filename] = contents + return { + "filename": file.filename, + "columns": columns, + "hostnames": hostnames, + "rows": len(df), + } + except Exception as e: + return {"error": str(e)} + + +# Simple in-memory cache for uploaded Excel files (cleared on restart) +excel_cache: dict[str, bytes] = {} + + +@app.get("/api/xmlbuilder/templates") +def xmlbuilder_list_templates(request: Request, _auth=Depends(require_auth)): + """List XML template files stored in xml_templates/.""" + files = sorted(TEMPLATES_DIR.glob("*.xml")) + result = [] + for f in files: + content = f.read_text(encoding="utf-8", errors="replace") + variables = _extract_template_variables(content) + result.append({ + "name": f.name, + "size_kb": f.stat().st_size // 1024, + "variables": variables, + }) + return {"templates": result} + + +@app.post("/api/xmlbuilder/upload-template") +async def xmlbuilder_upload_template(request: Request, file: UploadFile = File(...), _auth=Depends(require_auth)): + """Save an uploaded XML template into xml_templates/.""" + try: + contents = await file.read() + dest = TEMPLATES_DIR / file.filename + dest.write_bytes(contents) + text = contents.decode("utf-8", errors="replace") + variables = _extract_template_variables(text) + return {"name": file.filename, "variables": variables, "saved": True} + except Exception as e: + return {"error": str(e)} + + +@app.post("/api/xmlbuilder/generate/nocodb") +def xmlbuilder_generate_nocodb(request: Request, req: BuildRequest, _auth=Depends(require_auth)): + """Generate XMLs from NocoDB data and save to template_uploads/.""" + template_path = TEMPLATES_DIR / req.template_name + if not template_path.exists(): + return {"error": f"Template '{req.template_name}' not found in xml_templates/"} + template = template_path.read_text(encoding="utf-8") + results = build_xmls_from_nocodb(req.group, req.hostnames, template, req.default_value) + ok = sum(1 for r in results if r["success"]) + err = len(results) - ok + return {"results": results, "succeeded": ok, "failed": err} + + +@app.post("/api/xmlbuilder/generate/excel") +async def xmlbuilder_generate_excel( + request: Request, + hostnames: str = Form(...), # JSON array string + template_name: str = Form(...), + excel_filename: str = Form(...), # key into excel_cache + default_value: str = Form("NOTSET"), + excel_file: Optional[UploadFile] = File(None), + _auth=Depends(require_auth), +): + """Generate XMLs from Excel data and save to template_uploads/.""" + template_path = TEMPLATES_DIR / template_name + if not template_path.exists(): + return {"error": f"Template '{template_name}' not found in xml_templates/"} + template = template_path.read_text(encoding="utf-8") + + # Get Excel bytes — from fresh upload or cache + if excel_file: + excel_bytes = await excel_file.read() + excel_cache[excel_filename] = excel_bytes + elif excel_filename in excel_cache: + excel_bytes = excel_cache[excel_filename] + else: + return {"error": "Excel file not found. Please re-upload it."} + + host_list = json.loads(hostnames) + results = build_xmls_from_excel(excel_bytes, template, host_list, default_value) + ok = sum(1 for r in results if r["success"]) + err = len(results) - ok + return {"results": results, "succeeded": ok, "failed": err} + + +# ══════════════════════════════════════════════════════════════════════════════ +# Abort endpoint +# ══════════════════════════════════════════════════════════════════════════════ + +@app.post("/api/jobs/{job_id}/abort") +def abort_job(request: Request, job_id: str, _auth=Depends(require_auth)): + if job_id not in jobs: + return {"error": "Job not found"} + if jobs[job_id]["status"] == "done": + return {"error": "Job already finished"} + abort_flags.add(job_id) + jobs[job_id]["logs"].append( + f"[{datetime.now().strftime('%H:%M:%S')}] ⚠ Abort requested — stopping after current device..." + ) + return {"aborted": True, "job_id": job_id} + + +@app.post("/api/jobs/abort-all") +def abort_all_jobs(request: Request, _auth=Depends(require_auth)): + aborted = [] + for job_id, job in jobs.items(): + if job["status"] in ("running", "queued"): + abort_flags.add(job_id) + job["logs"].append( + f"[{datetime.now().strftime('%H:%M:%S')}] ⚠ Abort-all requested." + ) + aborted.append(job_id) + return {"aborted": aborted, "count": len(aborted)} + + +# ══════════════════════════════════════════════════════════════════════════════ +# AT Terminal — SSH command execution +# ══════════════════════════════════════════════════════════════════════════════ + +class ATRequest(BaseModel): + device_ids: list[str] = [] # empty = all devices in group + group: str = "All" + commands: list[str] # list of AT commands to send in order + ssh_username: str = "" + ssh_password: str = "" + concurrency: int = 3 + + +def _ssh_send_at_commands( + device: dict, + commands: list[str], + ssh_username: str, + ssh_password: str, + session_id: str, +) -> dict: + """ + Open an SSH session to one device on port 3223, send AT commands + one at a time, capture output, return result dict. + """ + device_id = device["id"] + ip = device["ip"] + username = ssh_username or device.get("username", "user") + password = ssh_password or device.get("password", "") + ts = lambda: datetime.now().strftime('%H:%M:%S') + + def log(msg, level="info"): + at_sessions[session_id]["logs"].append({ + "ts": ts(), "device": device_id, "msg": msg, "level": level + }) + + log(f"Connecting SSH → {ip}:{SSH_PORT}") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + ssh.connect( + ip, + port=SSH_PORT, + username=username, + password=password, + timeout=15, + look_for_keys=False, + allow_agent=False, + disabled_algorithms={"pubkeys": [], "keys": []}, + ) + except Exception as e: + log(f"✗ Connection failed: {str(e)[:80]}", "error") + return {"id": device_id, "success": False, "message": str(e)[:120], "outputs": []} + + outputs = [] + all_ok = True + + try: + # Use an interactive shell channel so the modem's AT prompt stays alive + chan = ssh.invoke_shell(term="vt100", width=200, height=50) + chan.settimeout(10) + + import time, socket + + # Drain any banner/login text the modem sends on connect + time.sleep(1.0) + buf = b"" + try: + while True: + chunk = chan.recv(4096) + if not chunk: + break + buf += chunk + except socket.timeout: + pass + + for cmd in commands: + log(f"→ {cmd}", "cmd") + chan.send(cmd + "\r") + time.sleep(0.8) + + response_buf = b"" + deadline = time.time() + 8 # max 8 s per command + try: + while time.time() < deadline: + if chan.recv_ready(): + chunk = chan.recv(4096) + if not chunk: + break + response_buf += chunk + # Stop early if we see OK or ERROR + decoded_so_far = response_buf.decode("utf-8", errors="replace") + if "\nOK" in decoded_so_far or "\nERROR" in decoded_so_far: + time.sleep(0.1) + break + else: + time.sleep(0.1) + except socket.timeout: + pass + + response = response_buf.decode("utf-8", errors="replace").strip() + # Strip the echoed command from the start if present + if response.startswith(cmd): + response = response[len(cmd):].strip() + + if response: + for line in response.splitlines(): + stripped = line.strip() + if stripped: + log(f" {stripped}", "data") + else: + log(" (no response)", "warn") + + outputs.append({"command": cmd, "output": response}) + + # If ATZ was sent the modem will reboot — nothing more to do + if cmd.strip().upper() == "ATZ": + log(" Modem rebooting.", "warn") + break + + except Exception as e: + log(f"✗ Session error: {str(e)[:80]}", "error") + all_ok = False + outputs.append({"command": "session", "output": str(e)}) + finally: + try: + chan.close() + except Exception: + pass + ssh.close() + + log(f"{'✓ Done' if all_ok else '✗ Finished with errors'}", "ok" if all_ok else "error") + return { + "id": device_id, "success": all_ok, + "message": "OK" if all_ok else "Errors — see log", + "outputs": outputs, + } + + +def _run_at_session(session_id: str, devices: list, commands: list, ssh_username: str, ssh_password: str, concurrency: int): + """Background thread: send AT commands to all selected devices.""" + import concurrent.futures + at_sessions[session_id]["status"] = "running" + results = [] + + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = { + executor.submit(_ssh_send_at_commands, d, commands, ssh_username, ssh_password, session_id): d + for d in devices + } + for future in concurrent.futures.as_completed(futures): + try: + results.append(future.result()) + except Exception as e: + d = futures[future] + results.append({"id": d["id"], "success": False, "message": str(e), "outputs": []}) + + at_sessions[session_id]["results"] = results + at_sessions[session_id]["status"] = "done" + at_sessions[session_id]["finished"] = datetime.now().isoformat() + + +@app.post("/api/at/send") +async def at_send(request: Request, req: ATRequest, background_tasks: BackgroundTasks, _auth=Depends(require_auth)): + """Start an AT command session against one or more devices.""" + if not req.commands or not any(c.strip() for c in req.commands): + return {"error": "No commands provided"} + commands = [c.strip() for c in req.commands if c.strip()] + + devices = load_devices(req.group) + if req.device_ids: + devices = [d for d in devices if d["id"] in req.device_ids] + if not devices: + return {"error": "No matching devices found"} + + session_id = f"at_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}" + at_sessions[session_id] = { + "status": "queued", + "started": datetime.now().isoformat(), + "finished": None, + "device_count": len(devices), + "commands": commands, + "logs": [], + "results": [], + } + + # Run in a background thread (paramiko is blocking/sync) + t = threading.Thread( + target=_run_at_session, + args=(session_id, devices, commands, req.ssh_username, req.ssh_password, req.concurrency), + daemon=True, + ) + t.start() + + return {"session_id": session_id, "device_count": len(devices)} + + +@app.get("/api/at/{session_id}/stream") +async def at_stream(request: Request, session_id: str, from_line: int = 0, _auth=Depends(require_auth)): + """SSE stream for AT session logs — same pattern as job streaming.""" + if session_id not in at_sessions: + return StreamingResponse( + iter([f'data: {json.dumps({"error": "session not found"})}\n\n']), + media_type="text/event-stream" + ) + + async def event_generator(): + sent = max(0, from_line) + while True: + if session_id not in at_sessions: + yield f'data: {json.dumps({"error": "session gone"})}\n\n' + break + sess = at_sessions[session_id] + logs = sess["logs"] + if len(logs) > sent: + for entry in logs[sent:]: + yield f"data: {json.dumps({'log': entry})}\n\n" + sent += 1 + if sess["status"] == "done": + yield f"data: {json.dumps({'done': True, 'results': sess['results']})}\n\n" + break + await asyncio.sleep(0.4) + + return StreamingResponse(event_generator(), media_type="text/event-stream") + + +@app.get("/api/at/{session_id}") +def at_get_session(request: Request, session_id: str, _auth=Depends(require_auth)): + if session_id not in at_sessions: + return {"error": "Session not found"} + return at_sessions[session_id] + + +# ══════════════════════════════════════════════════════════════════════════════ +# Auth routes — login / logout +# ══════════════════════════════════════════════════════════════════════════════ + +LOGIN_HTML = """ + + + + +RV50x Manager — Login + + + + +
+ + {error_block} +
+
+ + +
+
+ + +
+ +
+
HTTPS secured · Session expires in {hours}h
+
+ +""" + + +@app.get("/login", response_class=HTMLResponse) +def login_page(error: str = ""): + error_block = f'
⚠ {error}
' if error else "" + return HTMLResponse( + LOGIN_HTML + .replace("{error_block}", error_block) + .replace("{hours}", str(SESSION_HOURS)) + ) + + +@app.post("/auth/login") +async def auth_login(username: str = Form(...), password: str = Form(...)): + if username == APP_USERNAME and password == APP_PASSWORD: + token = _make_session_cookie() + resp = RedirectResponse(url="/", status_code=303) + resp.set_cookie( + key=SESSION_COOKIE, value=token, + httponly=True, secure=True, samesite="lax", + max_age=SESSION_HOURS * 3600, + ) + return resp + return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=303) + + +@app.get("/auth/logout") +def auth_logout(): + resp = RedirectResponse(url="/login", status_code=303) + resp.delete_cookie(SESSION_COOKIE) + return resp + + +# ══════════════════════════════════════════════════════════════════════════════ +# Serve frontend (auth protected) +# ══════════════════════════════════════════════════════════════════════════════ + +@app.get("/", response_class=HTMLResponse) +def index(request: Request, _auth=Depends(require_auth)): + html_path = SCRIPT_DIR / "index.html" + return HTMLResponse(html_path.read_text(encoding="utf-8")) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e4ef260 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,121 @@ +# ── RV50x Template Manager — Full Stack ──────────────────────────────────── +# +# Services: +# rv50x-manager FastAPI web app + Playwright +# nocodb NocoDB UI and API +# postgres PostgreSQL database for NocoDB +# +# Usage: +# Start: docker-compose up -d +# Stop: docker-compose stop +# Destroy: docker-compose down (data volumes preserved) +# Logs: docker-compose logs -f rv50x-manager +# +# Port mapping — change the LEFT number to use a different host port: +# rv50x-manager: http://host-ip:YOUR_PORT +# nocodb: http://host-ip:8090 +# postgres: not exposed (internal only) +# ─────────────────────────────────────────────────────────────────────────── + +services: + + # ── RV50x Template Manager ──────────────────────────────────────────────── + rv50x-manager: + build: + context: . + dockerfile: Dockerfile + container_name: rv50x-manager + restart: "no" # manual start only + ports: + - "8002:8000" # ← change YOUR_PORT to your chosen port + volumes: + # Bind mounts — files are directly accessible on the host at these paths. + # No sudo needed, no docker volume commands needed. + - /opt/rv50x-manager/template_downloads:/data/template_downloads + - /opt/rv50x-manager/template_uploads:/data/template_uploads + - /opt/rv50x-manager/xml_templates:/data/xml_templates + - /opt/rv50x-manager/certs:/certs:ro # SSL certificates (read-only) + environment: + # ── NocoDB connection ────────────────────────────────────────────── + # To use the built-in NocoDB from this stack, set NOCODB_URL to: + # http://nocodb:8080 + # To use an external NocoDB instance, set it to that URL instead. + NOCODB_URL: ${NOCODB_URL} + NOCODB_TOKEN: ${NOCODB_TOKEN} + NOCODB_BASE_ID: ${NOCODB_BASE_ID} + NOCODB_TABLE_ID: ${NOCODB_TABLE_ID} + NOCODB_VIEW_ID: ${NOCODB_VIEW_ID} + # ── SSL certificate paths (inside the container) ─────────────────── + SSL_CERT: /certs/cert.pem + SSL_KEY: /certs/key.pem + # ── Tunable timeouts (optional) ──────────────────────────────────── + PAGE_TIMEOUT: ${PAGE_TIMEOUT:-90000} + DOWNLOAD_TIMEOUT: ${DOWNLOAD_TIMEOUT:-120000} + UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120000} + MAX_RETRIES: ${MAX_RETRIES:-3} + APP_USERNAME: ${APP_USERNAME} + APP_PASSWORD: ${APP_PASSWORD} + SESSION_SECRET: ${SESSION_SECRET} + SESSION_HOURS: ${SESSION_HOURS:-8} + depends_on: + nocodb: + condition: service_healthy + networks: + - rv50x-net + + # ── NocoDB ─────────────────────────────────────────────────────────────── + nocodb: + image: nocodb/nocodb:latest + container_name: rv50x-nocodb + restart: "no" # manual start only + ports: + - "8090:8080" # NocoDB UI on host port 8090 + environment: + NC_DB: "pg://postgres:5432?u=nocodb&p=${POSTGRES_PASSWORD}&d=nocodb" + NC_AUTH_JWT_SECRET: ${NC_JWT_SECRET} + volumes: + - nocodb_data:/usr/app/data + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/health"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + networks: + - rv50x-net + + # ── PostgreSQL ─────────────────────────────────────────────────────────── + postgres: + image: postgres:16-alpine + container_name: rv50x-postgres + restart: "no" # manual start only + environment: + POSTGRES_USER: nocodb + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: nocodb + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U nocodb"] + interval: 5s + timeout: 3s + retries: 10 + networks: + - rv50x-net + # Postgres is intentionally NOT exposed on a host port. + # Only NocoDB can reach it via the internal network. + +# ── Named volumes — only postgres and nocodb use named volumes ─────────────── +# template_downloads, template_uploads, and xml_templates use bind mounts +# above so files are directly accessible in /opt/rv50x-manager/ on the host. +volumes: + postgres_data: + nocodb_data: + +# ── Internal network ──────────────────────────────────────────────────────── +networks: + rv50x-net: + driver: bridge diff --git a/env_example.txt b/env_example.txt new file mode 100644 index 0000000..c8bf9fc --- /dev/null +++ b/env_example.txt @@ -0,0 +1,31 @@ +# This is the internal Docker service name — leave as-is if using the +# built-in NocoDB from the stack. Change to your external NocoDB IP +# if you want to point at your existing instance instead. +NOCODB_URL=http://nocodb:8080 + +# Your NocoDB API token — get this from: +# NocoDB → Profile (bottom left) → Team & Settings → API Tokens +NOCODB_TOKEN=eWU_ilelaCtNy1JzC7vf41DokkqFOovcLHM0zVml + +# These three come from the NocoDB browser URL after you import your data. +# The URL looks like: http://host:8090/{org}/{BASE_ID}/{TABLE_ID}/{VIEW_ID}/... +NOCODB_BASE_ID=pdq96x915xt4a0m +NOCODB_TABLE_ID=mkewnr53ahqvnt9 +NOCODB_VIEW_ID=vwl7qvxo1xclvawz + +# Make up a strong password — you'll never type this manually, +# it's only used between the NocoDB and Postgres containers internally. +POSTGRES_PASSWORD=SomeLongStrongPassword123! + +# Any long random string — used to sign NocoDB login tokens. +# Generate one with: openssl rand -hex 32 +NC_JWT_SECRET=a1b2c3d4e5f6... + +The order of operations for first setup: + +Create .env with the passwords and secrets filled in — but leave the NocoDB IDs as placeholder values for now +Run docker-compose up -d to start the stack +Open NocoDB at http://host-ip:8090, create your account, import your CSV, recreate your views +Copy the real base/table/view IDs from the browser URL into .env +Run docker-compose restart rv50x-manager to pick up the new IDs +App is ready at http://host-ip:YOUR_PORT \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000..fef9e50 --- /dev/null +++ b/index.html @@ -0,0 +1,1672 @@ + + + + + +RV50x Template Manager + + + + + +
+ +
+
+ CONNECTED TO NOCODB + | + --:--:-- + + ⏻ +
+
+ + +
+
+ Appearance + +
+ + +
+
Presets
+
+
+
+ Cyan +
+
+
+ NocoDB +
+
+
+ Portainer +
+
+
+ Amber +
+
+
+ Green +
+
+
+ + +
+
Custom Accent Color
+
+
+ +
+
+ + +
+
Background Darkness
+
+ Darker + + Lighter + 50 +
+
+ + +
+
Font Style
+
+ + + +
+
+ + +
+ +
+
+ +
+ + + + + +
+
+ + + + + +
+ + +
+
+ + + + +
+
+ + + + + + + + + + + + +
HostnameIP AddressDeptLocationDL File
+
+
+ + +
+
+
+ JOB + + +
+ + + + + +
+
+ +
No job selected.
+ + +
+
+ + +
+
No jobs run yet.
+
+ + +
+
+ + +
+ + +
+
Data Source
+
+ + +
+ + +
+
Using live NocoDB data. Group filter applied from sidebar.
+
AVAILABLE VARIABLES
+
+
+ + + +
+ + +
+
XML Template
+
+ +
Upload new template (.xml)
+
+
+ + +
+
No templates yet
+
+ + +
+
Options
+
Default value for missing fields
+ +
+ + +
+
Select Devices
+
+ + +
+
Load a data source first
+
+
+ + +
+ +
+
+
+ + +
+
+
+ XML BUILDER + No template selected +
+
+
+ + +
+
TEMPLATE VARIABLES DETECTED
+
Select a template to see its variables
+
+ +
+
+ Select a data source, choose a template,
pick devices, then click Generate. +

+ Generated XMLs go directly to template_uploads/
+ and are ready to push to modems immediately. +
+
+
+
+
+ + +
+
+ + +
+ + +
+
SSH Credentials
+
+ + +
+
+ + +
+
PORT: 3223 (fixed)
+
+ + +
+
Parallel Sessions
+
+ Concurrency + 3 +
+ +
+ + +
+
Target Devices
+
+ + +
+ +
0 selected
+
+ + +
+
Loading devices…
+
+
+ + +
+ + +
+
+ AT COMMAND + one per line — all sent sequentially +
+ +
+ + + + + + + + + +
+
+ + + +
+
+ + +
+
// AT TERMINAL READY — SELECT DEVICES AND ENTER COMMAND
+
+ + + +
+
+
+ +
+
+ + + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7e2ef85 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +fastapi +uvicorn[standard] +playwright +pandas +openpyxl +paramiko +itsdangerous +python-multipart diff --git a/template_downloads/it-cm-00105.xml b/template_downloads/it-cm-00105.xml new file mode 100644 index 0000000..284c853 --- /dev/null +++ b/template_downloads/it-cm-00105.xml @@ -0,0 +1,3122 @@ + + diff --git a/template_downloads/it-cm-00120.xml b/template_downloads/it-cm-00120.xml new file mode 100644 index 0000000..ae0bb14 --- /dev/null +++ b/template_downloads/it-cm-00120.xml @@ -0,0 +1,3122 @@ + + diff --git a/template_uploads/it-cm-00090.xml b/template_uploads/it-cm-00090.xml new file mode 100644 index 0000000..7abf0df --- /dev/null +++ b/template_uploads/it-cm-00090.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00092.xml b/template_uploads/it-cm-00092.xml new file mode 100644 index 0000000..223ba5f --- /dev/null +++ b/template_uploads/it-cm-00092.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00094.xml b/template_uploads/it-cm-00094.xml new file mode 100644 index 0000000..e329436 --- /dev/null +++ b/template_uploads/it-cm-00094.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00096.xml b/template_uploads/it-cm-00096.xml new file mode 100644 index 0000000..c3b3103 --- /dev/null +++ b/template_uploads/it-cm-00096.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00098.xml b/template_uploads/it-cm-00098.xml new file mode 100644 index 0000000..288e524 --- /dev/null +++ b/template_uploads/it-cm-00098.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00105.xml b/template_uploads/it-cm-00105.xml new file mode 100644 index 0000000..e371758 --- /dev/null +++ b/template_uploads/it-cm-00105.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00116.xml b/template_uploads/it-cm-00116.xml new file mode 100644 index 0000000..b7cc96a --- /dev/null +++ b/template_uploads/it-cm-00116.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00120.xml b/template_uploads/it-cm-00120.xml new file mode 100644 index 0000000..482b9f9 --- /dev/null +++ b/template_uploads/it-cm-00120.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00130.xml b/template_uploads/it-cm-00130.xml new file mode 100644 index 0000000..0b99161 --- /dev/null +++ b/template_uploads/it-cm-00130.xml @@ -0,0 +1,2391 @@ + + diff --git a/template_uploads/it-cm-00137.xml b/template_uploads/it-cm-00137.xml new file mode 100644 index 0000000..ffa8e27 --- /dev/null +++ b/template_uploads/it-cm-00137.xml @@ -0,0 +1,2391 @@ + + diff --git a/xml_templates/cell_template_416RV50X_rev8.xml b/xml_templates/cell_template_416RV50X_rev8.xml new file mode 100644 index 0000000..e6d266d --- /dev/null +++ b/xml_templates/cell_template_416RV50X_rev8.xml @@ -0,0 +1,2391 @@ + +