feat: initial commit
This commit is contained in:
nikola
@@ -0,0 +1,14 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY backend/app /app/app
|
||||
COPY frontend /app/frontend
|
||||
COPY .env /app/.env
|
||||
|
||||
EXPOSE 8088
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8088"]
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .state import NetworkState
|
||||
|
||||
# dnsmasq style: "query[A] example.com from 192.168.88.10"
|
||||
DNS_QUERY_RE = re.compile(r"query\[[^\]]+\]\s+(?P<qname>\S+)\s+from\s+(?P<src_ip>\d+\.\d+\.\d+\.\d+)")
|
||||
|
||||
|
||||
async def tail_dns_log(state: NetworkState, path: str) -> None:
|
||||
if not path:
|
||||
return
|
||||
if not os.path.isfile(path):
|
||||
return
|
||||
|
||||
with open(path, "r", encoding="utf-8", errors="ignore") as f:
|
||||
f.seek(0, os.SEEK_END)
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
await asyncio.sleep(0.5)
|
||||
continue
|
||||
|
||||
m = DNS_QUERY_RE.search(line)
|
||||
if not m:
|
||||
continue
|
||||
src_ip = m.group("src_ip")
|
||||
qname = m.group("qname")
|
||||
await state.add_dns_event(src_ip=src_ip, qname=qname, answers=[], ts=datetime.now(timezone.utc))
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from .dnslog import tail_dns_log
|
||||
from .scanner import scan_subnet
|
||||
from .schemas import DhcpIngestEvent, DnsIngestEvent, FlowIngestEvent
|
||||
from .settings import settings
|
||||
from .state import NetworkState
|
||||
|
||||
state = NetworkState()
|
||||
|
||||
|
||||
async def scanner_loop() -> None:
|
||||
while True:
|
||||
active_ips = await scan_subnet(
|
||||
state=state,
|
||||
subnet_cidr=settings.scan_subnet,
|
||||
timeout_seconds=settings.scan_timeout_seconds,
|
||||
concurrency=settings.scan_concurrency,
|
||||
)
|
||||
await state.mark_offline_missing(active_ips)
|
||||
await asyncio.sleep(settings.scan_interval_seconds)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
tasks = [asyncio.create_task(scanner_loop(), name="scanner-loop")]
|
||||
if settings.dns_log_path:
|
||||
tasks.append(asyncio.create_task(tail_dns_log(state, settings.dns_log_path), name="dns-tail-loop"))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
for task in tasks:
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
app = FastAPI(title="Network Topology Wrapper", lifespan=lifespan)
|
||||
|
||||
static_dir = Path(__file__).resolve().parents[2] / "frontend"
|
||||
app.mount("/static", StaticFiles(directory=static_dir), name="static")
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def index() -> FileResponse:
|
||||
return FileResponse(static_dir / "index.html")
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/api/topology")
|
||||
def api_topology() -> dict:
|
||||
return state.topology_payload(
|
||||
core_router_ip=settings.core_router_ip,
|
||||
core_router_name=settings.core_router_name,
|
||||
core_switch_ip=settings.core_switch_ip,
|
||||
core_switch_name=settings.core_switch_name,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/ingest/dns")
|
||||
async def ingest_dns(event: DnsIngestEvent) -> dict[str, str]:
|
||||
await state.add_dns_event(
|
||||
src_ip=event.src_ip,
|
||||
qname=event.qname,
|
||||
answers=event.answers,
|
||||
ts=event.ts,
|
||||
)
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@app.post("/api/ingest/dhcp")
|
||||
async def ingest_dhcp(event: DhcpIngestEvent) -> dict[str, str]:
|
||||
await state.upsert_device(
|
||||
ip=event.ip,
|
||||
payload={
|
||||
"mac": event.mac,
|
||||
"hostname": event.hostname,
|
||||
"name": event.hostname or event.ip,
|
||||
"status": "online" if event.state != "offline" else "offline",
|
||||
"source": "dhcp",
|
||||
"type": "client",
|
||||
},
|
||||
)
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@app.post("/api/ingest/flow")
|
||||
async def ingest_flow(event: FlowIngestEvent) -> dict[str, str]:
|
||||
await state.add_flow_event(
|
||||
src_ip=event.src_ip,
|
||||
dst_ip=event.dst_ip,
|
||||
dst_host=event.dst_host,
|
||||
protocol=event.protocol,
|
||||
bytes_count=event.bytes_count,
|
||||
ts=event.ts,
|
||||
)
|
||||
return {"status": "accepted"}
|
||||
|
||||
|
||||
@app.get("/api/events/stream")
|
||||
async def events_stream() -> StreamingResponse:
|
||||
queue = state.subscribe()
|
||||
|
||||
async def stream():
|
||||
try:
|
||||
while True:
|
||||
event = await queue.get()
|
||||
payload = json.dumps(event, ensure_ascii=True)
|
||||
yield f"data: {payload}\n\n"
|
||||
finally:
|
||||
state.unsubscribe(queue)
|
||||
|
||||
return StreamingResponse(stream(), media_type="text/event-stream")
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
from typing import Any
|
||||
|
||||
from .state import NetworkState, utc_now
|
||||
|
||||
COMMON_PORTS = (22, 53, 80, 443, 8291)
|
||||
VENDOR_HINTS = {
|
||||
"mikrotik": "MikroTik",
|
||||
"cisco": "Cisco",
|
||||
"ubiquiti": "Ubiquiti",
|
||||
"raspberry": "Raspberry Pi",
|
||||
}
|
||||
|
||||
|
||||
def guess_device_type(hostname: str | None, open_port: int | None) -> str:
|
||||
if not hostname and open_port is None:
|
||||
return "unknown"
|
||||
|
||||
hn = (hostname or "").lower()
|
||||
if "rb4011" in hn or "router" in hn:
|
||||
return "router"
|
||||
if "crs" in hn or "switch" in hn or "slm" in hn:
|
||||
return "switch"
|
||||
if "cap" in hn or "ap" in hn or "wifi" in hn:
|
||||
return "ap"
|
||||
|
||||
if open_port == 8291:
|
||||
return "mikrotik"
|
||||
if open_port == 53:
|
||||
return "dns"
|
||||
return "client"
|
||||
|
||||
|
||||
def guess_vendor(hostname: str | None) -> str | None:
|
||||
if not hostname:
|
||||
return None
|
||||
hn = hostname.lower()
|
||||
for token, vendor in VENDOR_HINTS.items():
|
||||
if token in hn:
|
||||
return vendor
|
||||
return None
|
||||
|
||||
|
||||
async def probe_host(ip: str, timeout: float) -> tuple[bool, int | None]:
|
||||
for port in COMMON_PORTS:
|
||||
try:
|
||||
conn = asyncio.open_connection(ip, port)
|
||||
reader, writer = await asyncio.wait_for(conn, timeout=timeout)
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return True, port
|
||||
except Exception:
|
||||
continue
|
||||
return False, None
|
||||
|
||||
|
||||
def reverse_dns(ip: str) -> str | None:
|
||||
try:
|
||||
return socket.gethostbyaddr(ip)[0]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def scan_subnet(
|
||||
state: NetworkState,
|
||||
subnet_cidr: str,
|
||||
timeout_seconds: float,
|
||||
concurrency: int,
|
||||
) -> set[str]:
|
||||
network = ipaddress.ip_network(subnet_cidr, strict=False)
|
||||
active: set[str] = set()
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def one(ip_obj: Any) -> None:
|
||||
ip = str(ip_obj)
|
||||
async with sem:
|
||||
is_up, port = await probe_host(ip, timeout_seconds)
|
||||
if not is_up:
|
||||
return
|
||||
hostname = reverse_dns(ip)
|
||||
active.add(ip)
|
||||
await state.upsert_device(
|
||||
ip,
|
||||
{
|
||||
"hostname": hostname,
|
||||
"name": hostname or ip,
|
||||
"type": guess_device_type(hostname, port),
|
||||
"vendor": guess_vendor(hostname),
|
||||
"status": "online",
|
||||
"last_seen": utc_now(),
|
||||
"source": "scanner",
|
||||
},
|
||||
)
|
||||
|
||||
await asyncio.gather(*(one(ip) for ip in network.hosts()))
|
||||
return active
|
||||
@@ -0,0 +1,28 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DnsIngestEvent(BaseModel):
|
||||
src_ip: str
|
||||
qname: str
|
||||
answers: list[str] = Field(default_factory=list)
|
||||
ts: datetime | None = None
|
||||
|
||||
|
||||
class DhcpIngestEvent(BaseModel):
|
||||
ip: str
|
||||
mac: str | None = None
|
||||
hostname: str | None = None
|
||||
state: Literal["online", "offline", "lease"] = "lease"
|
||||
ts: datetime | None = None
|
||||
|
||||
|
||||
class FlowIngestEvent(BaseModel):
|
||||
src_ip: str
|
||||
dst_ip: str | None = None
|
||||
dst_host: str | None = None
|
||||
protocol: str | None = None
|
||||
bytes_count: int | None = None
|
||||
ts: datetime | None = None
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
app_host: str = "0.0.0.0"
|
||||
app_port: int = 8088
|
||||
|
||||
scan_subnet: str = "192.168.88.0/24"
|
||||
scan_interval_seconds: int = 15
|
||||
scan_timeout_seconds: float = 0.35
|
||||
scan_concurrency: int = 120
|
||||
|
||||
dns_log_path: str = ""
|
||||
|
||||
core_router_ip: str = "192.168.88.1"
|
||||
core_router_name: str = "RB4011"
|
||||
core_switch_ip: str = "192.168.88.2"
|
||||
core_switch_name: str = "CRS328-24P"
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,202 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class NetworkState:
|
||||
def __init__(self) -> None:
|
||||
self.devices: dict[str, dict[str, Any]] = {}
|
||||
self.dns_by_ip: dict[str, deque[dict[str, Any]]] = {}
|
||||
self.flows: deque[dict[str, Any]] = deque(maxlen=2000)
|
||||
self.events: deque[dict[str, Any]] = deque(maxlen=2000)
|
||||
self._subscribers: set[asyncio.Queue] = set()
|
||||
|
||||
async def publish(self, event: dict[str, Any]) -> None:
|
||||
self.events.append(event)
|
||||
stale: list[asyncio.Queue] = []
|
||||
for q in self._subscribers:
|
||||
try:
|
||||
q.put_nowait(event)
|
||||
except asyncio.QueueFull:
|
||||
stale.append(q)
|
||||
for q in stale:
|
||||
self._subscribers.discard(q)
|
||||
|
||||
def subscribe(self) -> asyncio.Queue:
|
||||
queue: asyncio.Queue = asyncio.Queue(maxsize=200)
|
||||
self._subscribers.add(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, queue: asyncio.Queue) -> None:
|
||||
self._subscribers.discard(queue)
|
||||
|
||||
async def upsert_device(self, ip: str, payload: dict[str, Any]) -> None:
|
||||
now = utc_now()
|
||||
prev = self.devices.get(ip)
|
||||
merged = {
|
||||
"ip": ip,
|
||||
"name": payload.get("name") or (prev or {}).get("name") or ip,
|
||||
"hostname": payload.get("hostname") or (prev or {}).get("hostname"),
|
||||
"mac": payload.get("mac") or (prev or {}).get("mac"),
|
||||
"vendor": payload.get("vendor") or (prev or {}).get("vendor"),
|
||||
"type": payload.get("type") or (prev or {}).get("type") or "unknown",
|
||||
"status": payload.get("status") or "online",
|
||||
"last_seen": payload.get("last_seen") or now,
|
||||
"first_seen": (prev or {}).get("first_seen") or now,
|
||||
"source": payload.get("source") or (prev or {}).get("source") or "scanner",
|
||||
}
|
||||
self.devices[ip] = merged
|
||||
|
||||
if prev is None:
|
||||
await self.publish({
|
||||
"kind": "device_discovered",
|
||||
"ip": ip,
|
||||
"device": self._serialize_device(merged),
|
||||
"ts": now.isoformat(),
|
||||
})
|
||||
elif prev.get("status") != "online":
|
||||
await self.publish({
|
||||
"kind": "device_online",
|
||||
"ip": ip,
|
||||
"device": self._serialize_device(merged),
|
||||
"ts": now.isoformat(),
|
||||
})
|
||||
|
||||
async def mark_offline_missing(self, active_ips: set[str]) -> None:
|
||||
now = utc_now()
|
||||
for ip, device in list(self.devices.items()):
|
||||
if ip in active_ips:
|
||||
continue
|
||||
if device.get("status") == "offline":
|
||||
continue
|
||||
if now - self._ensure_dt(device.get("last_seen")) < timedelta(seconds=45):
|
||||
continue
|
||||
device["status"] = "offline"
|
||||
await self.publish({
|
||||
"kind": "device_offline",
|
||||
"ip": ip,
|
||||
"device": self._serialize_device(device),
|
||||
"ts": now.isoformat(),
|
||||
})
|
||||
|
||||
async def add_dns_event(self, src_ip: str, qname: str, answers: list[str], ts: datetime | None = None) -> None:
|
||||
now = ts or utc_now()
|
||||
item = {"qname": qname, "answers": answers, "ts": now}
|
||||
bucket = self.dns_by_ip.setdefault(src_ip, deque(maxlen=200))
|
||||
bucket.append(item)
|
||||
|
||||
await self.upsert_device(src_ip, {"status": "online", "last_seen": now, "source": "dns"})
|
||||
await self.publish({
|
||||
"kind": "dns_query",
|
||||
"ip": src_ip,
|
||||
"qname": qname,
|
||||
"answers": answers,
|
||||
"ts": now.isoformat(),
|
||||
})
|
||||
|
||||
async def add_flow_event(
|
||||
self,
|
||||
src_ip: str,
|
||||
dst_ip: str | None,
|
||||
dst_host: str | None,
|
||||
protocol: str | None,
|
||||
bytes_count: int | None,
|
||||
ts: datetime | None = None,
|
||||
) -> None:
|
||||
now = ts or utc_now()
|
||||
flow = {
|
||||
"src_ip": src_ip,
|
||||
"dst_ip": dst_ip,
|
||||
"dst_host": dst_host,
|
||||
"protocol": protocol or "unknown",
|
||||
"bytes_count": bytes_count or 0,
|
||||
"ts": now,
|
||||
}
|
||||
self.flows.append(flow)
|
||||
|
||||
await self.upsert_device(src_ip, {"status": "online", "last_seen": now, "source": "flow"})
|
||||
await self.publish({
|
||||
"kind": "flow",
|
||||
"src_ip": src_ip,
|
||||
"dst_ip": dst_ip,
|
||||
"dst_host": dst_host,
|
||||
"protocol": protocol,
|
||||
"bytes_count": bytes_count,
|
||||
"ts": now.isoformat(),
|
||||
})
|
||||
|
||||
def topology_payload(self, core_router_ip: str, core_router_name: str, core_switch_ip: str, core_switch_name: str) -> dict[str, Any]:
|
||||
nodes: list[dict[str, Any]] = []
|
||||
edges: list[dict[str, Any]] = []
|
||||
|
||||
nodes.append({"id": "router", "label": core_router_name, "kind": "router", "ip": core_router_ip})
|
||||
nodes.append({"id": "switch", "label": core_switch_name, "kind": "switch", "ip": core_switch_ip})
|
||||
edges.append({"id": "e-router-switch", "source": "router", "target": "switch", "kind": "core"})
|
||||
|
||||
for ip, device in sorted(self.devices.items(), key=lambda x: tuple(int(p) for p in x[0].split("."))):
|
||||
node_id = f"dev-{ip}"
|
||||
label = device.get("hostname") or device.get("name") or ip
|
||||
nodes.append({
|
||||
"id": node_id,
|
||||
"label": label,
|
||||
"kind": device.get("type") or "unknown",
|
||||
"status": device.get("status") or "offline",
|
||||
"ip": ip,
|
||||
"vendor": device.get("vendor"),
|
||||
"last_seen": self._ensure_dt(device.get("last_seen")).isoformat(),
|
||||
})
|
||||
|
||||
parent = "router" if ip == core_router_ip else "switch"
|
||||
edges.append({"id": f"e-{parent}-{node_id}", "source": parent, "target": node_id, "kind": "lan"})
|
||||
|
||||
domain_nodes: dict[str, str] = {}
|
||||
for ip, entries in self.dns_by_ip.items():
|
||||
src_id = f"dev-{ip}"
|
||||
if src_id not in {n["id"] for n in nodes}:
|
||||
continue
|
||||
for item in list(entries)[-8:]:
|
||||
qname = item.get("qname", "")
|
||||
if not qname:
|
||||
continue
|
||||
domain_id = domain_nodes.get(qname)
|
||||
if domain_id is None:
|
||||
domain_id = f"domain-{len(domain_nodes)+1}"
|
||||
domain_nodes[qname] = domain_id
|
||||
nodes.append({"id": domain_id, "label": qname, "kind": "domain"})
|
||||
edges.append({
|
||||
"id": f"e-{src_id}-{domain_id}",
|
||||
"source": src_id,
|
||||
"target": domain_id,
|
||||
"kind": "dns",
|
||||
})
|
||||
|
||||
recent_events = list(self.events)[-40:]
|
||||
return {
|
||||
"generated_at": utc_now().isoformat(),
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"devices": [self._serialize_device(d) for d in self.devices.values()],
|
||||
"recent_events": recent_events,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _ensure_dt(value: Any) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
return utc_now()
|
||||
|
||||
@staticmethod
|
||||
def _serialize_device(device: dict[str, Any]) -> dict[str, Any]:
|
||||
out = dict(device)
|
||||
if isinstance(out.get("last_seen"), datetime):
|
||||
out["last_seen"] = out["last_seen"].isoformat()
|
||||
if isinstance(out.get("first_seen"), datetime):
|
||||
out["first_seen"] = out["first_seen"].isoformat()
|
||||
return out
|
||||
@@ -0,0 +1,4 @@
|
||||
fastapi==0.116.1
|
||||
uvicorn[standard]==0.35.0
|
||||
pydantic==2.11.7
|
||||
pydantic-settings==2.11.0
|
||||
Reference in New Issue
Block a user