commit 40d4679a59f8cd8327369cfd24be3e061bd76731 Author: dstephenson Date: Tue Apr 21 20:56:13 2026 +0000 Initial commit — LLDP network mapper Co-Authored-By: Claude Sonnet 4.6 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c44e3e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +data/ +backup/ +__pycache__/ +*.pyc +*.pyo +*.bak +*.swp +*.kate-swp +config.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a6543c2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +# Install graphviz for PNG rendering +RUN apt-get update && apt-get install -y --no-install-recommends \ + graphviz \ + openssh-client \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy and install requirements +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy all Python modules +COPY config.py db.py parser.py ssh_client.py scanner.py exports.py app.py nocodb_client.py ./ + +# Copy frontend template +COPY index.html ./templates/index.html + +# Create static and data dirs +RUN mkdir -p ./static /data + +# Data volume for SQLite + exports +VOLUME ["/data"] + +EXPOSE 5000 + +CMD ["python", "app.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..26b43fc --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# LLDP Network Mapper + +SSH-based network topology mapper for FS switches (and IOS-like CLIs). +Discovers switch topology via LLDP, stores in SQLite, and visualises with Cytoscape.js. + +## Quick Start + +### 1. Configure your switches + +Edit `mapper/config.py`: + +```python +SWITCHES = [ + "192.168.1.1", + "192.168.1.2", + # Add all switch management IPs +] + +SSH_USERNAME = "admin" +SSH_PASSWORD = "yourpassword" +``` + +### 2. Build and run + +```bash +docker-compose up --build +``` + +### 3. Open the UI + +Navigate to: http://localhost:5000 + +Click **Scan Now** to start discovery. + +--- + +## What it does + +- SSHs into each switch in parallel (up to 10 at once) +- Runs `show lldp neighbors` and `show ip interface brief` +- Parses neighbors, hostnames, management IPs, chassis IDs +- Stores everything in SQLite (`data/network.db`) +- De-duplicates bidirectional links automatically +- Renders an interactive Cytoscape.js topology diagram +- Exports to CSV, Mermaid (.md), and Graphviz PNG + +## Outputs + +All files written to `data/exports/`: + +| File | Purpose | +|------|---------| +| `topology.csv` | Switch links with hostnames and IPs | +| `topology.md` | Mermaid diagram (paste into any markdown viewer) | +| `topology.dot` | Graphviz source | +| `topology.png` | Rendered network diagram | + +## Auto-scan + +Toggle auto-scan on/off from the UI. Set interval (15 min to 6 hours). +State persists across container restarts. + +## Troubleshooting + +**Auth errors**: Check SSH_USERNAME / SSH_PASSWORD in config.py + +**Timeout errors**: Increase SSH_TIMEOUT in config.py (default: 30s) + +**Wrong device type**: If your FS switch uses a non-IOS CLI, try changing +DEVICE_TYPE to `"linux"` or `"generic"` in config.py + +**No management IP found**: The script looks for Vlan interfaces in +`show ip interface brief`. If your switch uses a different command, +edit `parser.py → parse_mgmt_ip_from_interfaces()` + +## Project Structure + +``` +lldp-mapper/ +├── mapper/ +│ ├── config.py # Switch IPs + credentials +│ ├── db.py # SQLite operations +│ ├── parser.py # LLDP output parser +│ ├── ssh_client.py # Netmiko SSH + parallel scan +│ ├── scanner.py # Orchestrator +│ └── exports.py # CSV / Mermaid / Graphviz +├── web/ +│ ├── app.py # Flask API + scheduler +│ └── templates/ +│ └── index.html # Cytoscape.js frontend +├── data/ # SQLite DB + exports (created at runtime) +├── Dockerfile +├── docker-compose.yml +└── README.md +``` diff --git a/app.py b/app.py new file mode 100644 index 0000000..47af3c9 --- /dev/null +++ b/app.py @@ -0,0 +1,228 @@ +# app.py - Flask backend +import logging +import threading +import os +from flask import Flask, jsonify, send_file, render_template, request +from apscheduler.schedulers.background import BackgroundScheduler + +import db +from db import save_node_positions, get_node_positions, clear_node_positions +import scanner +from config import EXPORTS_DIR + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__, template_folder='templates', static_folder='static') + +# --- Scheduler setup --- +apscheduler = BackgroundScheduler() +apscheduler.start() +_scheduler_job_id = "autoscan" + + +def _reschedule(interval_minutes): + if apscheduler.get_job(_scheduler_job_id): + apscheduler.remove_job(_scheduler_job_id) + apscheduler.add_job( + lambda: _trigger_scan_background(dept=None), + 'interval', + minutes=int(interval_minutes), + id=_scheduler_job_id, + replace_existing=True + ) + logger.info(f"Auto-scan scheduled every {interval_minutes} minutes") + + +def _trigger_scan_background(dept: str = None): + """Run scan in a background thread. dept=None → all switches.""" + if not scanner.scan_state["running"]: + t = threading.Thread(target=scanner.run_scan, kwargs={"dept": dept}, daemon=True) + t.start() + + +# Restore scheduler state on startup +db.init_db() +if db.get_setting("autoscan_enabled") == "true": + interval = db.get_setting("autoscan_interval") or "60" + _reschedule(interval) + + +# --- API Routes --- + +@app.route("/") +def index(): + return render_template("index.html") + + +@app.route("/api/scan", methods=["POST"]) +def api_scan(): + """Scan all active switches (no dept filter).""" + if scanner.scan_state["running"]: + return jsonify({"error": "Scan already running"}), 409 + _trigger_scan_background(dept=None) + return jsonify({"status": "started", "dept": None}) + + +@app.route("/api/scan/elec", methods=["POST"]) +def api_scan_elec(): + """Scan only ELEC department switches.""" + if scanner.scan_state["running"]: + return jsonify({"error": "Scan already running"}), 409 + _trigger_scan_background(dept="ELEC") + return jsonify({"status": "started", "dept": "ELEC"}) + + +@app.route("/api/scan/gw", methods=["POST"]) +def api_scan_gw(): + """Scan only GW department switches.""" + if scanner.scan_state["running"]: + return jsonify({"error": "Scan already running"}), 409 + _trigger_scan_background(dept="GW") + return jsonify({"status": "started", "dept": "GW"}) + + +@app.route("/api/status") +def api_status(): + state = dict(scanner.scan_state) + last = db.get_last_scan() + state["last_scan_log"] = dict(last) if last else None + state["autoscan_enabled"] = db.get_setting("autoscan_enabled") == "true" + state["autoscan_interval"] = int(db.get_setting("autoscan_interval") or 60) + return jsonify(state) + + +@app.route("/api/switches") +def api_switches(): + return jsonify(db.get_all_switches()) + + +@app.route("/api/links") +def api_links(): + return jsonify(db.get_all_links()) + + +@app.route("/api/topology") +def api_topology(): + switches = {s["chassis_id"]: s for s in db.get_all_switches()} + links = db.get_all_links() + + nodes = [] + for chassis_id, sw in switches.items(): + nodes.append({ + "data": { + "id": chassis_id, + "label": sw.get("hostname") or chassis_id, + "hostname": sw.get("hostname", ""), + "mgmt_ip": sw.get("mgmt_ip", ""), + "description": sw.get("description", ""), + "chassis_id": chassis_id, + "last_seen": sw.get("last_seen", ""), + } + }) + + edges = [] + for link in links: + edge_id = f"{link['chassis_a']}:{link['port_a']}__{link['chassis_b']}:{link['port_b']}" + hostname_a = switches.get(link['chassis_a'], {}).get('hostname', link['chassis_a']) + hostname_b = switches.get(link['chassis_b'], {}).get('hostname', link['chassis_b']) + edges.append({ + "data": { + "id": edge_id, + "source": link["chassis_a"], + "target": link["chassis_b"], + "port_a": link["port_a"], + "port_b": link["port_b"], + "hostname_a": hostname_a, + "hostname_b": hostname_b, + } + }) + + return jsonify({"nodes": nodes, "edges": edges}) + + +@app.route("/api/settings", methods=["POST"]) +def api_settings(): + data = request.json + enabled = data.get("autoscan_enabled", False) + interval = int(data.get("autoscan_interval", 60)) + + db.set_setting("autoscan_enabled", "true" if enabled else "false") + db.set_setting("autoscan_interval", str(interval)) + + if enabled: + _reschedule(interval) + else: + if apscheduler.get_job(_scheduler_job_id): + apscheduler.remove_job(_scheduler_job_id) + logger.info("Auto-scan disabled") + + return jsonify({"autoscan_enabled": enabled, "autoscan_interval": interval}) + + +@app.route("/api/layout", methods=["POST"]) +def api_save_layout(): + data = request.json + positions = data.get("positions", []) + if not positions: + return jsonify({"error": "No positions provided"}), 400 + # Normalize 'id' -> 'chassis_id' so JS can send either key + normalized = [{"chassis_id": p.get("chassis_id") or p.get("id"), "x": p["x"], "y": p["y"]} for p in positions] + save_node_positions(normalized) + return jsonify({"saved": len(positions)}) + + +@app.route("/api/layout", methods=["DELETE"]) +def api_clear_layout(): + clear_node_positions() + return jsonify({"status": "cleared"}) + + +@app.route("/api/layout", methods=["GET"]) +def api_get_layout(): + positions = get_node_positions() + return jsonify({"has_custom_layout": len(positions) > 0, "count": len(positions)}) + +@app.route("/api/layout/positions", methods=["GET"]) +def api_get_layout_positions(): + """Return saved positions as list of {id, x, y} for Cytoscape.""" + positions = get_node_positions() + return jsonify([{"id": cid, "x": xy[0], "y": xy[1]} for cid, xy in positions.items()]) + + +@app.route("/api/switches/clear-stale", methods=["POST"]) +def api_clear_stale_switches(): + """Remove switches not seen in the last scan.""" + last = db.get_last_scan() + if not last: + return jsonify({"error": "No scan has been run yet"}), 400 + since = last["started_at"] + deleted = db.clear_stale_switches(since) + return jsonify({"deleted": deleted}) + +@app.route("/api/export/csv") +def api_export_csv(): + path = os.path.join(EXPORTS_DIR, "topology.csv") + if not os.path.exists(path): + return jsonify({"error": "No export found, run a scan first"}), 404 + return send_file(path, as_attachment=True, download_name="topology.csv") + + +@app.route("/api/export/mermaid") +def api_export_mermaid(): + path = os.path.join(EXPORTS_DIR, "topology.md") + if not os.path.exists(path): + return jsonify({"error": "No export found, run a scan first"}), 404 + return send_file(path, as_attachment=True, download_name="topology.md") + + +@app.route("/api/export/png") +def api_export_png(): + path = os.path.join(EXPORTS_DIR, "topology.png") + if not os.path.exists(path): + return jsonify({"error": "No PNG found, run a scan first"}), 404 + return send_file(path, as_attachment=True, download_name="topology.png") + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5000, debug=False) diff --git a/config.py.example b/config.py.example new file mode 100644 index 0000000..229f33d --- /dev/null +++ b/config.py.example @@ -0,0 +1,20 @@ +# config.py - Central configuration +# Copy this file to config.py and fill in your values. + +# ─── NocoDB ─────────────────────────────────────────────────────────────────── +NOCODB_URL = "http://192.168.x.x:8080" +NOCODB_TOKEN = "your-nocodb-api-token" + +# ─── SSH ────────────────────────────────────────────────────────────────────── +SSH_USERNAME = "admin" +SSH_PASSWORD = "your-ssh-password" +SSH_PORT = 22 +SSH_TIMEOUT = 30 + +# Netmiko device type for FS switches (IOS-like CLI) +DEVICE_TYPE = "cisco_ios" + +# ─── Output paths ───────────────────────────────────────────────────────────── +DATA_DIR = "/data" +DB_PATH = "/data/network.db" +EXPORTS_DIR = "/data/exports" diff --git a/db.py b/db.py new file mode 100644 index 0000000..5fa36e5 --- /dev/null +++ b/db.py @@ -0,0 +1,312 @@ +# db.py - SQLite database operations +import sqlite3 +import re +import os +import logging +from config import DB_PATH + +logger = logging.getLogger(__name__) + + +def get_conn(): + conn = sqlite3.connect(DB_PATH, check_same_thread=False, timeout=10) + conn.row_factory = sqlite3.Row + return conn + + +def init_db(): + os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) + conn = get_conn() + c = conn.cursor() + + c.execute(""" + CREATE TABLE IF NOT EXISTS switches ( + chassis_id TEXT PRIMARY KEY, + hostname TEXT, + mgmt_ip TEXT, + description TEXT, + last_seen TEXT + ) + """) + + c.execute(""" + CREATE TABLE IF NOT EXISTS links ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chassis_a TEXT, + port_a TEXT, + chassis_b TEXT, + port_b TEXT, + UNIQUE(chassis_a, port_a, chassis_b, port_b) + ) + """) + + c.execute(""" + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT + ) + """) + + c.execute(""" + CREATE TABLE IF NOT EXISTS scan_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT, + finished_at TEXT, + status TEXT, + switches_ok INTEGER DEFAULT 0, + switches_fail INTEGER DEFAULT 0 + ) + """) + + c.execute("INSERT OR IGNORE INTO settings VALUES ('autoscan_enabled', 'false')") + c.execute("INSERT OR IGNORE INTO settings VALUES ('autoscan_interval', '60')") + + c.execute( + "CREATE TABLE IF NOT EXISTS node_positions " + "(chassis_id TEXT PRIMARY KEY, x REAL, y REAL)" + ) + + conn.commit() + conn.close() + + +def upsert_switch(chassis_id, hostname, mgmt_ip, description): + conn = get_conn() + conn.execute(""" + INSERT INTO switches (chassis_id, hostname, mgmt_ip, description, last_seen) + VALUES (?, ?, ?, ?, datetime('now')) + ON CONFLICT(chassis_id) DO UPDATE SET + hostname = excluded.hostname, + mgmt_ip = CASE WHEN excluded.mgmt_ip != '' AND excluded.mgmt_ip NOT LIKE '%.%.%.%' = 0 + THEN excluded.mgmt_ip ELSE mgmt_ip END, + description = CASE WHEN excluded.description != '' THEN excluded.description ELSE description END, + last_seen = excluded.last_seen + """, (chassis_id, hostname, mgmt_ip, description)) + conn.commit() + conn.close() + + +def upsert_link(chassis_a, port_a, chassis_b, port_b): + # Normalize order so A→B and B→A are treated as the same link + if chassis_a > chassis_b: + chassis_a, chassis_b = chassis_b, chassis_a + port_a, port_b = port_b, port_a + conn = get_conn() + # Check both orderings before inserting + existing = conn.execute(""" + SELECT id FROM links WHERE + (chassis_a=? AND port_a=? AND chassis_b=? AND port_b=?) OR + (chassis_a=? AND port_a=? AND chassis_b=? AND port_b=?) + """, (chassis_a, port_a, chassis_b, port_b, + chassis_b, port_b, chassis_a, port_a)).fetchone() + if not existing: + conn.execute(""" + INSERT OR IGNORE INTO links (chassis_a, port_a, chassis_b, port_b) + VALUES (?, ?, ?, ?) + """, (chassis_a, port_a, chassis_b, port_b)) + conn.commit() + conn.close() + + +def clear_stale_switches(since_dt): + """Remove switches not seen since the given datetime string.""" + conn = get_conn() + conn.execute("DELETE FROM switches WHERE last_seen < ? OR last_seen IS NULL", (since_dt,)) + deleted = conn.execute("SELECT changes()").fetchone()[0] + conn.commit() + conn.close() + return deleted + +def clear_links(): + conn = get_conn() + conn.execute("DELETE FROM links") + conn.commit() + conn.close() + + +def _is_mac(chassis): + return bool(re.match(r'^[0-9A-Fa-f]{2}[-:][0-9A-Fa-f]{2}', chassis)) + + +def _is_real_ip(val): + return bool(re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', val or '')) + + +def merge_duplicate_switches(): + """ + After a scan, some switches may have two records: + - One with MAC chassis_id (seen as neighbor by another switch) + - One with IP chassis_id (directly scanned but chassis extraction failed) + This function merges them: keeps MAC as canonical, fixes links, removes IP record. + """ + conn = get_conn() + + dupes = conn.execute(""" + SELECT hostname, COUNT(*) as cnt, GROUP_CONCAT(chassis_id) as ids, + GROUP_CONCAT(mgmt_ip) as ips + FROM switches + GROUP BY hostname + HAVING cnt > 1 + """).fetchall() + + merged = 0 + for row in dupes: + ids = row['ids'].split(',') + ips_raw = row['ips'].split(',') + + mac_ids = [i for i in ids if _is_mac(i)] + ip_ids = [i for i in ids if not _is_mac(i)] + real_ips = [v for v in ips_raw if _is_real_ip(v)] + + if not mac_ids or not ip_ids: + continue + + canonical = mac_ids[0] + best_ip = real_ips[0] if real_ips else '' + + # Update canonical with best mgmt_ip + if best_ip: + conn.execute("UPDATE switches SET mgmt_ip=? WHERE chassis_id=?", + (best_ip, canonical)) + + # Repoint links + for old_id in ip_ids: + conn.execute("UPDATE links SET chassis_a=? WHERE chassis_a=?", (canonical, old_id)) + conn.execute("UPDATE links SET chassis_b=? WHERE chassis_b=?", (canonical, old_id)) + conn.execute("DELETE FROM switches WHERE chassis_id=?", (old_id,)) + + merged += 1 + logger.info(f"Merged {row['hostname']}: {ip_ids} -> {canonical} (mgmt_ip={best_ip})") + + # Remove any self-links or exact duplicate links created by repointing + # Remove duplicate links - normalize chassis+port order before grouping + conn.execute(""" + DELETE FROM links WHERE id NOT IN ( + SELECT MIN(id) FROM links + GROUP BY + CASE WHEN chassis_a < chassis_b THEN chassis_a ELSE chassis_b END, + CASE WHEN chassis_a < chassis_b THEN chassis_b ELSE chassis_a END, + CASE WHEN chassis_a < chassis_b THEN port_a ELSE port_b END, + CASE WHEN chassis_a < chassis_b THEN port_b ELSE port_a END + ) + """) + conn.execute("DELETE FROM links WHERE chassis_a = chassis_b") + + conn.commit() + conn.close() + logger.info(f"merge_duplicate_switches: merged {merged} hostname groups") + + +def get_all_switches(): + conn = get_conn() + rows = conn.execute("SELECT * FROM switches ORDER BY hostname").fetchall() + conn.close() + return [dict(r) for r in rows] + + +def get_all_links(): + conn = get_conn() + rows = conn.execute(""" + SELECT l.*, + sa.hostname as hostname_a, sa.mgmt_ip as ip_a, + sb.hostname as hostname_b, sb.mgmt_ip as ip_b + FROM links l + LEFT JOIN switches sa ON l.chassis_a = sa.chassis_id + LEFT JOIN switches sb ON l.chassis_b = sb.chassis_id + """).fetchall() + conn.close() + return [dict(r) for r in rows] + + +def get_setting(key): + conn = get_conn() + row = conn.execute("SELECT value FROM settings WHERE key=?", (key,)).fetchone() + conn.close() + return row["value"] if row else None + + +def set_setting(key, value): + conn = get_conn() + conn.execute("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)", (key, str(value))) + conn.commit() + conn.close() + + +def log_scan_start(): + conn = get_conn() + c = conn.cursor() + c.execute("INSERT INTO scan_log (started_at, status) VALUES (datetime('now'), 'running')") + scan_id = c.lastrowid + conn.commit() + conn.close() + return scan_id + + +def log_scan_finish(scan_id, ok_count, fail_count): + conn = get_conn() + conn.execute(""" + UPDATE scan_log SET finished_at=datetime('now'), status='done', + switches_ok=?, switches_fail=? WHERE id=? + """, (ok_count, fail_count, scan_id)) + conn.commit() + conn.close() + + +def get_last_scan(): + conn = get_conn() + row = conn.execute( + "SELECT * FROM scan_log ORDER BY id DESC LIMIT 1" + ).fetchone() + conn.close() + return dict(row) if row else None + + +def deduplicate_links(): + """Remove duplicate bidirectional links. Always run after a scan.""" + conn = get_conn() + before = conn.execute("SELECT COUNT(*) FROM links").fetchone()[0] + conn.execute(""" + DELETE FROM links WHERE id NOT IN ( + SELECT MIN(id) FROM links + GROUP BY + CASE WHEN chassis_a < chassis_b THEN chassis_a ELSE chassis_b END, + CASE WHEN chassis_a < chassis_b THEN chassis_b ELSE chassis_a END, + CASE WHEN chassis_a < chassis_b THEN port_a ELSE port_b END, + CASE WHEN chassis_a < chassis_b THEN port_b ELSE port_a END + ) + """) + conn.execute("DELETE FROM links WHERE chassis_a = chassis_b") + conn.commit() + after = conn.execute("SELECT COUNT(*) FROM links").fetchone()[0] + conn.close() + logger.info(f"deduplicate_links: {before} -> {after} links") + + +def save_node_positions(positions): + """Save node positions from Cytoscape. positions = list of {chassis_id, x, y}""" + conn = get_conn() + conn.execute("DELETE FROM node_positions") + for p in positions: + conn.execute( + "INSERT INTO node_positions (chassis_id, x, y) VALUES (?, ?, ?)", + (p['chassis_id'], float(p['x']), float(p['y'])) + ) + conn.commit() + conn.close() + logging.getLogger(__name__).info(f"Saved {len(positions)} node positions") + + +def get_node_positions(): + """Returns dict of chassis_id -> (x, y), empty if none saved.""" + conn = get_conn() + rows = conn.execute("SELECT chassis_id, x, y FROM node_positions").fetchall() + conn.close() + return {r["chassis_id"]: (r["x"], r["y"]) for r in rows} + + +def clear_node_positions(): + conn = get_conn() + conn.execute("DELETE FROM node_positions") + conn.commit() + conn.close() + logging.getLogger(__name__).info("Node positions cleared") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a2a666c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + lldp-mapper: + build: . + container_name: lldp-mapper + ports: + - "5000:5000" + volumes: + - ./data:/data + - ./config.py:/app/config.py + - ./index.html:/app/templates/index.html:ro + restart: unless-stopped + environment: + - PYTHONUNBUFFERED=1 diff --git a/exports.py b/exports.py new file mode 100644 index 0000000..e97b58c --- /dev/null +++ b/exports.py @@ -0,0 +1,190 @@ +# exports.py - Generate CSV, Mermaid, and Graphviz outputs from DB +import csv +import os +import logging +from config import EXPORTS_DIR +from db import get_all_switches, get_all_links + +logger = logging.getLogger(__name__) + + +def _ensure_exports_dir(): + os.makedirs(EXPORTS_DIR, exist_ok=True) + + +def export_csv(): + """Export links + switch info to CSV.""" + _ensure_exports_dir() + path = os.path.join(EXPORTS_DIR, "topology.csv") + switches = {s['chassis_id']: s for s in get_all_switches()} + links = get_all_links() + + with open(path, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow([ + 'Switch A Hostname', 'Switch A IP', 'Switch A Chassis', + 'Port A', + 'Switch B Hostname', 'Switch B IP', 'Switch B Chassis', + 'Port B' + ]) + for link in links: + sw_a = switches.get(link['chassis_a'], {}) + sw_b = switches.get(link['chassis_b'], {}) + writer.writerow([ + sw_a.get('hostname', link['chassis_a']), + sw_a.get('mgmt_ip', ''), + link['chassis_a'], + link['port_a'], + sw_b.get('hostname', link['chassis_b']), + sw_b.get('mgmt_ip', ''), + link['chassis_b'], + link['port_b'], + ]) + + logger.info(f"CSV exported to {path}") + return path + + +def export_mermaid(): + """Export topology as clean Mermaid diagram with reference tables.""" + _ensure_exports_dir() + path = os.path.join(EXPORTS_DIR, "topology.md") + switches = {s['chassis_id']: s for s in get_all_switches()} + links = get_all_links() + + def node_id(sw, chassis_id): + return sw.get('hostname', chassis_id).replace('-', '_').replace('.', '_') + + lines = [ + "# Network Topology", + "", + "```mermaid", + "graph LR", + ] + + # Node definitions - hostname only as label + for chassis_id, sw in sorted(switches.items(), key=lambda x: x[1].get('hostname','')): + nid = node_id(sw, chassis_id) + hostname = sw.get('hostname', chassis_id) + lines.append(f' {nid}["{hostname}"]') + + lines.append("") + + # Edge definitions - portA --> portB as label + for link in links: + sw_a = switches.get(link['chassis_a'], {}) + sw_b = switches.get(link['chassis_b'], {}) + id_a = node_id(sw_a, link['chassis_a']) + id_b = node_id(sw_b, link['chassis_b']) + lines.append(f' {id_a} -- "{link["port_a"]} to {link["port_b"]}" --> {id_b}') + + lines.append("```") + lines.append("") + + # Switch reference table + lines.append("## Switch Reference") + lines.append("") + lines.append("| Hostname | Management IP | Chassis ID | Last Seen |") + lines.append("|----------|---------------|------------|-----------|") + for sw in sorted(switches.values(), key=lambda x: x.get('hostname', '')): + lines.append( + f"| {sw.get('hostname','')} " + f"| {sw.get('mgmt_ip','')} " + f"| {sw.get('chassis_id','')} " + f"| {sw.get('last_seen','')} |" + ) + + lines.append("") + + # Link reference table + lines.append("## Link Reference") + lines.append("") + lines.append("| Switch A | Port A | Switch B | Port B |") + lines.append("|----------|--------|----------|--------|") + for link in links: + sw_a = switches.get(link['chassis_a'], {}) + sw_b = switches.get(link['chassis_b'], {}) + lines.append( + f"| {sw_a.get('hostname', link['chassis_a'])} " + f"| {link['port_a']} " + f"| {sw_b.get('hostname', link['chassis_b'])} " + f"| {link['port_b']} |" + ) + + with open(path, 'w') as f: + f.write('\n'.join(lines)) + + logger.info(f"Mermaid exported to {path}") + return path + + +def export_graphviz(): + """Export topology as Graphviz DOT and render to PNG.""" + _ensure_exports_dir() + dot_path = os.path.join(EXPORTS_DIR, "topology.dot") + png_path = os.path.join(EXPORTS_DIR, "topology.png") + + switches = {s['chassis_id']: s for s in get_all_switches()} + links = get_all_links() + + def nid(chassis): + sw = switches.get(chassis, {}) + return sw.get('hostname', chassis).replace('-', '_').replace('.', '_') + + dot = [] + dot.append('digraph network {') + dot.append(' rankdir=LR;') + dot.append(' bgcolor="white";') + dot.append(' pad=0.8;') + dot.append(' nodesep=0.6;') + dot.append(' ranksep=1.5;') + dot.append(' node [shape=box, style="filled,rounded", fillcolor="#1a3a6e", fontcolor=white, fontname="Helvetica Bold", fontsize=12, margin="0.3,0.2", width=2.2];') + dot.append(' edge [fontname="Helvetica", fontsize=9, fontcolor="#444444", color="#1a3a6e", penwidth=1.5, dir=none];') + dot.append('') + + for chassis_id, sw in switches.items(): + n = nid(chassis_id) + hostname = sw.get('hostname', chassis_id) + mgmt_ip = sw.get('mgmt_ip', '') + label = f"{hostname}\n{mgmt_ip}" if mgmt_ip else hostname + dot.append(f' {n} [label="{label}"];') + + dot.append('') + + for link in links: + a = nid(link["chassis_a"]) + b = nid(link["chassis_b"]) + pa = link["port_a"] + pb = link["port_b"] + dot.append(f' {a} -> {b} [taillabel="{pa}", headlabel="{pb}"];') + + dot.append('}') + + with open(dot_path, 'w') as f: + f.write('\n'.join(dot)) + + try: + import subprocess + result = subprocess.run( + ['dot', '-Tpng', '-Gdpi=150', dot_path, '-o', png_path], + capture_output=True, text=True, timeout=30 + ) + if result.returncode == 0: + logger.info(f"Graphviz PNG exported to {png_path}") + return png_path + else: + logger.error(f"Graphviz render failed: {result.stderr}") + return dot_path + except FileNotFoundError: + logger.warning("graphviz not found") + return dot_path + except Exception as e: + logger.error(f"Graphviz error: {e}") + return dot_path + + +def run_all_exports(): + csv_path = export_csv() + md_path = export_mermaid() + png_path = export_graphviz() + return {"csv": csv_path, "mermaid": md_path, "graphviz": png_path} diff --git a/files.zip b/files.zip new file mode 100644 index 0000000..51890a6 Binary files /dev/null and b/files.zip differ diff --git a/index.html b/index.html new file mode 100644 index 0000000..165ebcb --- /dev/null +++ b/index.html @@ -0,0 +1,900 @@ + + + + + +LLDP Network Mapper + + + + + + +
+ + +
+ + No scan yet + +
+
+ Auto-scan + + +
+ + + + +
+
+ + +
+
+
+ + +
+ + +
+
+ +
+ + + +

No topology data

+

Run a scan to discover your network

+
+ +
+ + + + + + + +
+
+ + +
+
Network Detail
+
+ + + +
+ + + +

Click a switch or link
to see details

+
+ + +
+ +
+ + + +
+
+
+ + +
+ + Scanning... +
+ + + + + diff --git a/nocodb_client.py b/nocodb_client.py new file mode 100644 index 0000000..c80046e --- /dev/null +++ b/nocodb_client.py @@ -0,0 +1,69 @@ +# nocodb_client.py - Fetch switch list from NocoDB "Fiber Switches" base +import logging +import requests +from config import NOCODB_URL, NOCODB_TOKEN + +logger = logging.getLogger(__name__) + +BASE_ID = "p1h7iyzjjbsnfv5" +TABLE_ID = "md03qoibd5fz839" + +HEADERS = { + "xc-token": NOCODB_TOKEN, + "Content-Type": "application/json", +} + + +def get_switches(dept: str = None) -> list[dict]: + """ + Return list of switch dicts from NocoDB (Active=YES only). + dept: None = all, "ELEC" or "GW" to filter by department. + Raises RuntimeError if NocoDB is unreachable or returns no rows. + """ + where = "(Active,eq,YES)" + if dept: + where = f"(Active,eq,YES)~and(Dept,eq,{dept})" + + switches = [] + offset = 0 + limit = 100 + + while True: + r = requests.get( + f"{NOCODB_URL}/api/v1/db/data/noco/{BASE_ID}/{TABLE_ID}", + headers=HEADERS, + params={"limit": limit, "offset": offset, "where": where}, + timeout=15, + ) + if r.status_code != 200: + raise RuntimeError(f"NocoDB error {r.status_code}: {r.text[:200]}") + + data = r.json() + for row in data.get("list", []): + ip = (row.get("IP") or "").strip() + if not ip: + continue + switches.append({ + "ip": ip, + "hostname": (row.get("Hostname") or "").strip(), + "location": (row.get("Location") or "").strip(), + "model": (row.get("Model") or "").strip(), + "manufacturer": (row.get("Manufacturer") or "FS").strip(), + "dept": (row.get("Dept") or "").strip(), + "asset_tag": (row.get("Asset Tag") or "").strip(), + }) + + if data.get("pageInfo", {}).get("isLastPage", True): + break + offset += limit + + if not switches: + raise RuntimeError(f"NocoDB returned 0 active switches (dept={dept})") + + logger.info(f"NocoDB: loaded {len(switches)} switches" + (f" (dept={dept})" if dept else "")) + return switches + + +def get_switch_ips(dept: str = None) -> list[str]: + """Convenience wrapper — returns just the IP list.""" + return [s["ip"] for s in get_switches(dept=dept)] diff --git a/parser.py b/parser.py new file mode 100644 index 0000000..3660276 --- /dev/null +++ b/parser.py @@ -0,0 +1,88 @@ +# parser.py - Parse 'show lldp neighbors' output from FS switches +import re + + +def shorten_interface(iface): + """GigabitEthernet 1/9 -> Gi1/9, TenGigabitEthernet 1/1 -> Te1/1 etc.""" + replacements = [ + (r'GigabitEthernet\s*', 'Gi'), + (r'TenGigabitEthernet\s*', 'Te'), + (r'TwentyFiveGigE\s*', 'Twe'), + (r'FortyGigabitEthernet\s*', 'Fo'), + (r'HundredGigE\s*', 'Hu'), + (r'FastEthernet\s*', 'Fa'), + (r'Ethernet\s*', 'Eth'), + (r'mgmt\s*', 'mgmt'), + ] + for pattern, short in replacements: + iface = re.sub(pattern, short, iface, flags=re.IGNORECASE) + return iface.strip() + + +def parse_lldp_neighbors(raw_output, local_chassis_id, local_hostname, local_mgmt_ip): + """ + Parse raw 'show lldp neighbors' output from an FS switch. + + Returns: + neighbors: list of dicts with parsed neighbor info + local_info: dict with this switch's details (enriched from LLDP data if needed) + """ + neighbors = [] + + # Split output into per-neighbor blocks (blank line separated) + # Each block starts with "Local Interface" + blocks = re.split(r'\n\s*\n', raw_output.strip()) + + for block in blocks: + if not block.strip(): + continue + if 'Local Interface' not in block and 'Local Port' not in block: + continue + + neighbor = {} + + def extract(pattern, text, default=''): + m = re.search(pattern, text, re.IGNORECASE) + return m.group(1).strip() if m else default + + neighbor['local_port'] = shorten_interface(extract(r'Local Interface\s*:\s*(.+)', block)) + neighbor['chassis_id'] = extract(r'Chassis ID\s*:\s*(.+)', block) + neighbor['port_id'] = extract(r'Port ID\s*:\s*(.+)', block) + neighbor['port_desc'] = shorten_interface(extract(r'Port Description\s*:\s*(.+)', block)) + neighbor['system_name'] = extract(r'System Name\s*:\s*(.+)', block) + neighbor['system_desc'] = extract(r'System Description\s*:\s*(.+)', block) + # FS switches report Management Address as MAC (e.g. '64-9D-99-AA-50-B0 (Other)') + # or as IP (e.g. '10.214.0.192'). Extract only valid IPv4. + raw_mgmt = extract(r'Management Address\s*:\s*([\d\.A-Fa-f\-:]+)', block) + ipv4_match = re.search(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', raw_mgmt) + neighbor['mgmt_ip'] = ipv4_match.group(1) if ipv4_match else '' + neighbor['capabilities'] = extract(r'System Capabilities\s*:\s*(.+)', block) + + # Only include bridge/switch neighbors (skip phones, APs listed as endpoints) + if neighbor['chassis_id'] and neighbor['system_name']: + # Use port_desc as remote port if available, fallback to port_id + neighbor['remote_port'] = neighbor['port_desc'] if neighbor['port_desc'] else neighbor['port_id'] + neighbors.append(neighbor) + + return neighbors + + +def parse_hostname_from_prompt(prompt_line): + """Extract hostname from CLI prompt like 'ls-vhls-sw01#'""" + m = re.match(r'^([A-Za-z0-9_\-]+)[>#]', prompt_line.strip()) + return m.group(1) if m else None + + +def parse_mgmt_ip_from_interfaces(raw_output): + """ + Parse 'show ip interface brief' to find management VLAN IP. + Looks for Vlan interfaces with an IP assigned. + Returns first Vlan IP found (typically the management VLAN). + """ + lines = raw_output.splitlines() + for line in lines: + # Match lines like: Vlan100 192.168.1.10 YES ... + m = re.match(r'\s*(Vlan\S+)\s+([\d\.]+)\s+', line, re.IGNORECASE) + if m and not m.group(2).startswith('0.0.0.0'): + return m.group(2) + return None diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9e2e1c8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +flask==3.0.3 +netmiko==4.3.0 +apscheduler==3.10.4 +graphviz==0.20.3 +paramiko==3.4.0 +requests diff --git a/scanner.py b/scanner.py new file mode 100644 index 0000000..c9061fe --- /dev/null +++ b/scanner.py @@ -0,0 +1,122 @@ +# scanner.py - Orchestrates the full scan pipeline +import logging +from nocodb_client import get_switch_ips +from ssh_client import scan_all_switches +from db import ( + upsert_switch, upsert_link, clear_links, + log_scan_start, log_scan_finish, merge_duplicate_switches +) +from exports import run_all_exports + +logger = logging.getLogger(__name__) + +# Global scan state (shared with Flask via import) +scan_state = { + "running": False, + "done": 0, + "total": 0, + "current_ip": None, + "current_hostname": None, + "ok": 0, + "fail": 0, + "errors": [], + "last_scan": None, + "dept_filter": None, # None = all, "ELEC" or "GW" = dept-only +} + + +def run_scan(dept: str = None): + """ + Full scan: fetch switch IPs from NocoDB → SSH all → parse → store → export. + dept: None = all active switches, "ELEC" or "GW" = dept-filtered. + """ + global scan_state + + if scan_state["running"]: + logger.warning("Scan already running, skipping.") + return + + # Fetch switch list from NocoDB (or fallback) + switches = get_switch_ips(dept=dept) + + if not switches: + logger.error("NocoDB returned no switches — aborting scan.") + scan_state["running"] = False + return + logger.error("No switches returned from NocoDB, aborting scan.") + return + + scan_state.update({ + "running": True, + "done": 0, + "total": len(switches), + "current_ip": None, + "current_hostname": None, + "ok": 0, + "fail": 0, + "errors": [], + "dept_filter": dept, + }) + + scan_id = log_scan_start() + clear_links() # Fresh start for links each scan + + def on_progress(done, total, ip, result): + scan_state["done"] = done + scan_state["total"] = total + scan_state["current_ip"] = ip + + if result["success"]: + scan_state["ok"] += 1 + scan_state["current_hostname"] = result.get("hostname", ip) + + upsert_switch( + chassis_id=result["chassis_id"], + hostname=result["hostname"], + mgmt_ip=result["mgmt_ip"], + description=result.get("description", ""), + ) + + for neighbor in result["neighbors"]: + if neighbor.get("chassis_id") and neighbor.get("system_name"): + upsert_switch( + chassis_id=neighbor["chassis_id"], + hostname=neighbor["system_name"], + mgmt_ip=neighbor.get("mgmt_ip", ""), + description=neighbor.get("system_desc", ""), + ) + upsert_link( + chassis_a=result["chassis_id"], + port_a=neighbor["local_port"], + chassis_b=neighbor["chassis_id"], + port_b=neighbor["remote_port"], + ) + else: + scan_state["fail"] += 1 + scan_state["errors"].append({ + "ip": ip, + "error": result.get("error", "Unknown error") + }) + + scan_all_switches(switches, progress_callback=on_progress) + + try: + merge_duplicate_switches() + except Exception as e: + logger.error(f"Merge error: {e}") + + try: + run_all_exports() + except Exception as e: + logger.error(f"Export error: {e}") + + log_scan_finish(scan_id, scan_state["ok"], scan_state["fail"]) + + from datetime import datetime + scan_state["last_scan"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + scan_state["running"] = False + scan_state["current_ip"] = None + scan_state["current_hostname"] = None + scan_state["dept_filter"] = None + + logger.info(f"Scan complete. OK: {scan_state['ok']}, Failed: {scan_state['fail']}") diff --git a/ssh_client.py b/ssh_client.py new file mode 100644 index 0000000..8c53d06 --- /dev/null +++ b/ssh_client.py @@ -0,0 +1,133 @@ +# ssh_client.py - Netmiko SSH connections to FS switches +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from netmiko import ConnectHandler, NetmikoTimeoutException, NetmikoAuthenticationException +from config import SSH_USERNAME, SSH_PASSWORD, SSH_PORT, SSH_TIMEOUT, DEVICE_TYPE +from parser import parse_lldp_neighbors, parse_mgmt_ip_from_interfaces + +logger = logging.getLogger(__name__) + + +def connect_and_query(ip): + """ + SSH into a switch, run LLDP + interface commands. + Returns dict with switch info and neighbors, or error info. + """ + device = { + "device_type": DEVICE_TYPE, + "host": ip, + "username": SSH_USERNAME, + "password": SSH_PASSWORD, + "port": SSH_PORT, + "timeout": SSH_TIMEOUT, + "global_delay_factor": 2, + "fast_cli": False, + "conn_timeout": SSH_TIMEOUT, +# "ssh_config_file": None, +# "disabled_algorithms": {}, +# "transport": "paramiko", + } + + # FS switches use legacy ssh-rsa keys — must be explicitly allowed + import paramiko + _orig_connect = ConnectHandler.__init__ + + try: + logger.info(f"Connecting to {ip}...") + + # Patch paramiko transport to allow legacy key algorithms + import paramiko.transport as _pt + _orig_preferred_keys = _pt.Transport._preferred_keys + _pt.Transport._preferred_keys = ( + "ssh-rsa", "rsa-sha2-256", "rsa-sha2-512", + "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", "ssh-ed25519", + ) + + with ConnectHandler(**device) as conn: + # Restore after connect + _pt.Transport._preferred_keys = _orig_preferred_keys + + # Get hostname from prompt + hostname = conn.find_prompt().replace('#', '').replace('>', '').strip() + + # Get LLDP neighbors + lldp_output = conn.send_command("show lldp neighbors", read_timeout=30) + + # Get management IP from interface brief + intf_output = conn.send_command("show ip interface brief", read_timeout=30) + mgmt_ip = parse_mgmt_ip_from_interfaces(intf_output) or ip + + # Get chassis ID from LLDP local info + local_info_output = conn.send_command("show lldp local-information", read_timeout=30) + chassis_id = _extract_local_chassis(local_info_output) or ip + + # Parse neighbors + neighbors = parse_lldp_neighbors(lldp_output, chassis_id, hostname, mgmt_ip) + + # Get system description + sys_desc = _extract_system_desc(local_info_output) + + logger.info(f" {hostname} ({ip}): {len(neighbors)} neighbors found") + + return { + "success": True, + "ip": ip, + "hostname": hostname, + "chassis_id": chassis_id, + "mgmt_ip": mgmt_ip, + "description": sys_desc, + "neighbors": neighbors, + } + + except NetmikoAuthenticationException: + logger.error(f"Auth failed for {ip}") + return {"success": False, "ip": ip, "error": "Authentication failed"} + except NetmikoTimeoutException: + logger.error(f"Timeout connecting to {ip}") + return {"success": False, "ip": ip, "error": "Connection timed out"} + except Exception as e: + logger.error(f"Error on {ip}: {e}") + return {"success": False, "ip": ip, "error": str(e)} + + +def _extract_local_chassis(output): + import re + m = re.search(r'Chassis ID\s*:\s*([0-9A-Fa-f\-:]+)', output) + return m.group(1).strip() if m else None + + +def _extract_system_desc(output): + import re + m = re.search(r'System Description\s*:\s*(.+)', output) + return m.group(1).strip() if m else '' + + +def scan_all_switches(ip_list, progress_callback=None, max_workers=10): + """ + Scan all switches in parallel. + progress_callback(done, total, current_ip, result) called after each switch. + Returns list of results. + """ + results = [] + total = len(ip_list) + done = 0 + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_ip = {executor.submit(connect_and_query, ip): ip for ip in ip_list} + + for future in as_completed(future_to_ip): + ip = future_to_ip[future] + try: + result = future.result() + except Exception as e: + result = {"success": False, "ip": ip, "error": str(e)} + import time; time.sleep(2) # 2s delay between switches to avoid RADIUS lockout + + results.append(result) + done += 1 + + if progress_callback: + progress_callback(done, total, ip, result) + + return results diff --git a/sync_to_netbox.py b/sync_to_netbox.py new file mode 100644 index 0000000..864aa7a --- /dev/null +++ b/sync_to_netbox.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +""" +sync_to_netbox.py +Reads the lldp-mapper SQLite DB and syncs devices/links to NetBox. +Additive only - adds new devices/cables, never deletes anything. +Run manually: python3 sync_to_netbox.py +""" + +import sqlite3 +import requests +import logging + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +# --- Configuration --- +DB_PATH = "/opt/lldp-mapper/data/network.db" + +NETBOX_URL = "http://192.168.16.130:8001" +NETBOX_TOKEN = "nbt_T6aq9XpwNFQG.HtZziXuSATgabbeagWKk3vEhc2Ask1EMV210PWMM" + +SITE_NAME = "Field Sites" +DEVICE_ROLE = "Access Switch" +MANUFACTURER = "FS" +DEVICE_TYPE = "FS Switch" + +# --- Helpers --- + +NB_HEADERS = { + "Authorization": f"Bearer {NETBOX_TOKEN}", + "Content-Type": "application/json", + "Accept": "application/json", +} + + +def nb_get(path, params=None): + r = requests.get(f"{NETBOX_URL}/api/{path}", headers=NB_HEADERS, params=params) + r.raise_for_status() + return r.json() + + +def nb_post(path, data): + r = requests.post(f"{NETBOX_URL}/api/{path}", headers=NB_HEADERS, json=data) + if r.status_code not in (200, 201): + logger.error(f"NetBox POST {path} failed: {r.status_code} {r.text}") + return None + return r.json() + + +def nb_get_or_create(path, lookup_params, create_data): + results = nb_get(path, params=lookup_params).get("results", []) + if results: + return results[0] + logger.info(f"Creating {path}: {create_data.get('name', create_data)}") + return nb_post(path, create_data) + + +# --- Read from SQLite --- + +def load_db(): + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + switches = [dict(r) for r in conn.execute("SELECT * FROM switches ORDER BY hostname").fetchall()] + links = [dict(r) for r in conn.execute(""" + SELECT l.*, sa.hostname as hn_a, sb.hostname as hn_b + FROM links l + LEFT JOIN switches sa ON l.chassis_a = sa.chassis_id + LEFT JOIN switches sb ON l.chassis_b = sb.chassis_id + """).fetchall()] + conn.close() + return switches, links + + +# --- NetBox Sync --- + +def ensure_site(): + return nb_get_or_create( + "dcim/sites/", + {"name": SITE_NAME}, + {"name": SITE_NAME, "slug": SITE_NAME.lower().replace(" ", "-")} + ) + + +def ensure_manufacturer(): + return nb_get_or_create( + "dcim/manufacturers/", + {"name": MANUFACTURER}, + {"name": MANUFACTURER, "slug": MANUFACTURER.lower()} + ) + + +def ensure_device_type(manufacturer_id): + return nb_get_or_create( + "dcim/device-types/", + {"slug": "fs-switch"}, + {"model": DEVICE_TYPE, "slug": "fs-switch", "manufacturer": manufacturer_id} + ) + + +def ensure_device_role(): + return nb_get_or_create( + "dcim/device-roles/", + {"name": DEVICE_ROLE}, + {"name": DEVICE_ROLE, "slug": "access-switch", "color": "2196f3"} + ) + + +def ensure_device(switch, site_id, device_type_id, role_id): + hostname = switch["hostname"] or switch["mgmt_ip"] + results = nb_get("dcim/devices/", params={"name": hostname}).get("results", []) + if results: + logger.info(f" Device exists: {hostname}") + return results[0] + logger.info(f" Creating device: {hostname}") + return nb_post("dcim/devices/", { + "name": hostname, + "site": site_id, + "device_type": device_type_id, + "role": role_id, + "status": "active", + "comments": f"Chassis ID: {switch['chassis_id']}\nDiscovered by lldp-mapper", + }) + + +def ensure_interface(device_id, port_name): + results = nb_get("dcim/interfaces/", params={ + "device_id": device_id, + "name": port_name + }).get("results", []) + if results: + return results[0] + return nb_post("dcim/interfaces/", { + "device": device_id, + "name": port_name, + "type": "1000base-t", + }) + + +def ensure_ip(switch, device_id): + if not switch.get("mgmt_ip"): + return + ip_addr = f"{switch['mgmt_ip']}/24" + results = nb_get("ipam/ip-addresses/", params={"address": ip_addr}).get("results", []) + if results: + ip_obj = results[0] + else: + logger.info(f" Creating IP: {ip_addr}") + ip_obj = nb_post("ipam/ip-addresses/", { + "address": ip_addr, + "status": "active", + }) + if not ip_obj: + return + requests.patch( + f"{NETBOX_URL}/api/dcim/devices/{device_id}/", + headers=NB_HEADERS, + json={"primary_ip4": ip_obj["id"]} + ) + + +def ensure_cable(device_map, link): + hn_a = link["hn_a"] + hn_b = link["hn_b"] + port_a = link["port_a"] + port_b = link["port_b"] + + if hn_a not in device_map or hn_b not in device_map: + logger.warning(f" Skipping cable {hn_a}:{port_a} <-> {hn_b}:{port_b} — device not in NetBox") + return + + dev_a = device_map[hn_a] + dev_b = device_map[hn_b] + + iface_a = ensure_interface(dev_a["id"], port_a) + iface_b = ensure_interface(dev_b["id"], port_b) + + if not iface_a or not iface_b: + return + + result_a = nb_get("dcim/interfaces/", params={"device_id": dev_a["id"], "name": port_a}).get("results", []) + if result_a and result_a[0].get("cable"): + logger.info(f" Cable already exists: {hn_a}:{port_a} <-> {hn_b}:{port_b}") + return + + logger.info(f" Creating cable: {hn_a}:{port_a} <-> {hn_b}:{port_b}") + nb_post("dcim/cables/", { + "a_terminations": [{"object_type": "dcim.interface", "object_id": iface_a["id"]}], + "b_terminations": [{"object_type": "dcim.interface", "object_id": iface_b["id"]}], + "status": "connected", + }) + + +def sync_netbox(switches, links): + logger.info("=== Syncing to NetBox ===") + + site = ensure_site() + manufacturer = ensure_manufacturer() + device_type = ensure_device_type(manufacturer["id"]) + role = ensure_device_role() + + site_id = site["id"] + device_type_id = device_type["id"] + role_id = role["id"] + + device_map = {} + + for sw in switches: + hostname = sw["hostname"] or sw["mgmt_ip"] + logger.info(f"Processing device: {hostname}") + device = ensure_device(sw, site_id, device_type_id, role_id) + if device: + device_map[hostname] = device + ensure_ip(sw, device["id"]) + + logger.info(f"Devices synced: {len(device_map)}") + + logger.info("Processing cables...") + for link in links: + ensure_cable(device_map, link) + + logger.info("NetBox sync complete.") + + +# --- Main --- + +if __name__ == "__main__": + switches, links = load_db() + logger.info(f"Loaded {len(switches)} switches and {len(links)} links from DB") + sync_netbox(switches, links) + logger.info("=== All done ===") diff --git a/sync_to_netbox_full.py b/sync_to_netbox_full.py new file mode 100644 index 0000000..298223d --- /dev/null +++ b/sync_to_netbox_full.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +sync_to_netbox_full.py +Full sync - clears all cables in NetBox and re-creates from lldp-mapper DB. +Use this after topology changes (moved switches, new connections, removed links). + +For a safe additive-only sync, use sync_to_netbox.py instead. +""" + +import sqlite3 +import requests +import logging + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +# --- Configuration --- +DB_PATH = "/opt/lldp-mapper/data/network.db" + +NETBOX_URL = "http://192.168.16.130:8001" +NETBOX_TOKEN = "nbt_T6aq9XpwNFQG.HtZziXuSATgabbeagWKk3vEhc2Ask1EMV210PWMM" + +SITE_NAME = "Field Sites" +DEVICE_ROLE = "Access Switch" +MANUFACTURER = "FS" +DEVICE_TYPE = "FS Switch" + +# --- Helpers --- + +NB_HEADERS = { + "Authorization": f"Bearer {NETBOX_TOKEN}", + "Content-Type": "application/json", + "Accept": "application/json", +} + + +def nb_get(path, params=None): + r = requests.get(f"{NETBOX_URL}/api/{path}", headers=NB_HEADERS, params=params) + r.raise_for_status() + return r.json() + + +def nb_post(path, data): + r = requests.post(f"{NETBOX_URL}/api/{path}", headers=NB_HEADERS, json=data) + if r.status_code not in (200, 201): + logger.error(f"NetBox POST {path} failed: {r.status_code} {r.text}") + return None + return r.json() + + +def nb_delete(path): + r = requests.delete(f"{NETBOX_URL}/api/{path}", headers=NB_HEADERS) + return r.status_code == 204 + + +def nb_get_or_create(path, lookup_params, create_data): + results = nb_get(path, params=lookup_params).get("results", []) + if results: + return results[0] + logger.info(f"Creating {path}: {create_data.get('name', create_data)}") + return nb_post(path, create_data) + + +# --- Read from SQLite --- + +def load_db(): + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + switches = [dict(r) for r in conn.execute("SELECT * FROM switches ORDER BY hostname").fetchall()] + links = [dict(r) for r in conn.execute(""" + SELECT l.*, sa.hostname as hn_a, sb.hostname as hn_b + FROM links l + LEFT JOIN switches sa ON l.chassis_a = sa.chassis_id + LEFT JOIN switches sb ON l.chassis_b = sb.chassis_id + """).fetchall()] + conn.close() + return switches, links + + +# --- NetBox Sync --- + +def ensure_site(): + return nb_get_or_create( + "dcim/sites/", + {"name": SITE_NAME}, + {"name": SITE_NAME, "slug": SITE_NAME.lower().replace(" ", "-")} + ) + + +def ensure_manufacturer(): + return nb_get_or_create( + "dcim/manufacturers/", + {"name": MANUFACTURER}, + {"name": MANUFACTURER, "slug": MANUFACTURER.lower()} + ) + + +def ensure_device_type(manufacturer_id): + return nb_get_or_create( + "dcim/device-types/", + {"slug": "fs-switch"}, + {"model": DEVICE_TYPE, "slug": "fs-switch", "manufacturer": manufacturer_id} + ) + + +def ensure_device_role(): + return nb_get_or_create( + "dcim/device-roles/", + {"name": DEVICE_ROLE}, + {"name": DEVICE_ROLE, "slug": "access-switch", "color": "2196f3"} + ) + + +def ensure_device(switch, site_id, device_type_id, role_id): + hostname = switch["hostname"] or switch["mgmt_ip"] + results = nb_get("dcim/devices/", params={"name": hostname}).get("results", []) + if results: + logger.info(f" Device exists: {hostname}") + return results[0] + logger.info(f" Creating device: {hostname}") + return nb_post("dcim/devices/", { + "name": hostname, + "site": site_id, + "device_type": device_type_id, + "role": role_id, + "status": "active", + "comments": f"Chassis ID: {switch['chassis_id']}\nDiscovered by lldp-mapper", + }) + + +def ensure_interface(device_id, port_name): + results = nb_get("dcim/interfaces/", params={ + "device_id": device_id, + "name": port_name + }).get("results", []) + if results: + return results[0] + return nb_post("dcim/interfaces/", { + "device": device_id, + "name": port_name, + "type": "1000base-t", + }) + + +def ensure_ip(switch, device_id): + if not switch.get("mgmt_ip"): + return + ip_addr = f"{switch['mgmt_ip']}/24" + results = nb_get("ipam/ip-addresses/", params={"address": ip_addr}).get("results", []) + if results: + ip_obj = results[0] + else: + logger.info(f" Creating IP: {ip_addr}") + ip_obj = nb_post("ipam/ip-addresses/", { + "address": ip_addr, + "status": "active", + }) + if not ip_obj: + return + requests.patch( + f"{NETBOX_URL}/api/dcim/devices/{device_id}/", + headers=NB_HEADERS, + json={"primary_ip4": ip_obj["id"]} + ) + + +def clear_all_cables(): + logger.info("Clearing all existing cables from NetBox...") + count = 0 + while True: + results = nb_get("dcim/cables/", params={"limit": 50}).get("results", []) + if not results: + break + for cable in results: + nb_delete(f"dcim/cables/{cable['id']}/") + count += 1 + logger.info(f"Deleted {count} cables.") + + +def create_cable(device_map, link): + hn_a = link["hn_a"] + hn_b = link["hn_b"] + port_a = link["port_a"] + port_b = link["port_b"] + + if hn_a not in device_map or hn_b not in device_map: + logger.warning(f" Skipping cable {hn_a}:{port_a} <-> {hn_b}:{port_b} — device not in NetBox") + return + + dev_a = device_map[hn_a] + dev_b = device_map[hn_b] + + iface_a = ensure_interface(dev_a["id"], port_a) + iface_b = ensure_interface(dev_b["id"], port_b) + + if not iface_a or not iface_b: + return + + logger.info(f" Creating cable: {hn_a}:{port_a} <-> {hn_b}:{port_b}") + nb_post("dcim/cables/", { + "a_terminations": [{"object_type": "dcim.interface", "object_id": iface_a["id"]}], + "b_terminations": [{"object_type": "dcim.interface", "object_id": iface_b["id"]}], + "status": "connected", + }) + + +def sync_netbox(switches, links): + logger.info("=== Syncing to NetBox (FULL - cables will be cleared and re-created) ===") + + site = ensure_site() + manufacturer = ensure_manufacturer() + device_type = ensure_device_type(manufacturer["id"]) + role = ensure_device_role() + + site_id = site["id"] + device_type_id = device_type["id"] + role_id = role["id"] + + device_map = {} + + for sw in switches: + hostname = sw["hostname"] or sw["mgmt_ip"] + logger.info(f"Processing device: {hostname}") + device = ensure_device(sw, site_id, device_type_id, role_id) + if device: + device_map[hostname] = device + ensure_ip(sw, device["id"]) + + logger.info(f"Devices synced: {len(device_map)}") + + clear_all_cables() + + logger.info("Re-creating cables from current scan data...") + for link in links: + create_cable(device_map, link) + + logger.info("NetBox sync complete.") + + +# --- Main --- + +if __name__ == "__main__": + logger.info("*** FULL SYNC — existing cables will be deleted and re-created ***") + switches, links = load_db() + logger.info(f"Loaded {len(switches)} switches and {len(links)} links from DB") + sync_netbox(switches, links) + logger.info("=== All done ===")