analyze performance upgrade

.env.example

@@ -0,0 +1,45 @@
+# IoT PreTester 4.6 Demo - local configuration template
+# Copy to .env and set real values for SECRET_KEY and DJANGO_SUPERUSER_PASSWORD.
+#   cp .env.example .env
+
+# --- Django core -------------------------------------------------------------
+DEBUG=False
+# Required when DEBUG=False. Generate one with:
+#   python3 -c "import secrets; print(secrets.token_urlsafe(64))"
+SECRET_KEY=CHANGE_ME
+ALLOWED_HOSTS=localhost,127.0.0.1
+
+# --- Demo administrator ------------------------------------------------------
+DJANGO_SUPERUSER_USERNAME=admin
+DJANGO_SUPERUSER_EMAIL=admin@example.com
+DJANGO_SUPERUSER_PASSWORD=CHANGE_ME
+INIT_DATABASE=true
+
+# --- CORS --------------------------------------------------------------------
+CORS_ALLOW_ALL_ORIGINS=False
+CORS_ALLOWED_ORIGINS=http://localhost:3000
+
+# --- Ports -------------------------------------------------------------------
+BACKEND_PORT=8000
+FRONTEND_PORT=3000
+
+# --- Capture analysis limits -------------------------------------------------
+# Packets to inspect per capture. 0 reads the capture completely; a 250 MB
+# capture takes well under a minute with the tshark-based extractor.
+CAPTURE_PACKET_LIMIT=0
+# Retained clear-text payload entries (encrypted flows are never retained).
+CAPTURE_MAX_TEXT_PAYLOADS=50000
+CAPTURE_MAX_PAYLOAD_CHARS=16384
+# Extra expected provider domains (comma-separated suffixes). Traffic to these
+# is reported at lower severity, never hidden. Example: vendor-cloud.com,iot.example
+ANALYZER_BENIGN_DOMAINS=
+
+# --- Gunicorn / logging ------------------------------------------------------
+GUNICORN_WORKERS=4
+GUNICORN_TIMEOUT=600
+# debug shows every detector step and the tshark command lines in the logs.
+LOG_LEVEL=info
+
+# --- Frontend ----------------------------------------------------------------
+NODE_ENV=production
+NEXT_PUBLIC_API_URL=http://localhost:8000/api

.gitignore

@@ -0,0 +1,29 @@
+# Local configuration and secrets
+.env
+.env.local
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+venv/
+
+# Django runtime data
+django_project/staticfiles/
+django_project/db.sqlite3
+data/
+logs/
+files/uploads/
+uploads/
+
+# Node / Next.js
+node_modules/
+frontend/.next/
+frontend/out/
+frontend/node_modules/
+
+# OS / editor
+.DS_Store
+.idea/
+.vscode/

ARCHITECTURE.md

@@ -12,16 +12,16 @@ IoT PreTester 4.6 Demo analyzes existing `.pcap` and `.pcapng` files and turns p
 | Django REST backend | API, validation, persistence, user/device administration, and integration with the analysis core |
 | `analyzer` package | Django-independent capture extraction, detection, scoring, provenance, and report rendering |
 | SQLite | Demo persistence for devices, files, findings, extracted artifacts, and analysis status |
-| tshark/pyshark | Protocol dissection and normalized capture extraction |
+| tshark | Protocol dissection; driven directly through two streaming `tshark -T ek` passes |
 | nmap/hashcat | Optional, explicitly initiated endpoint probing and offline hash recovery |
 
 ## Analysis flow
 
 1. The frontend sends the capture and metadata to the authenticated Django API.
 2. Django validates the upload, stores it in a dedicated volume, and creates the database records.
-3. `extract_pyshark.py` parses the capture once into a normalized `CaptureContext`.
+3. `extract_tshark.py` reads the capture into a normalized `CaptureContext`: one metadata pass over every packet, then one payload pass restricted to unencrypted flows. The device IP entered at upload time is stored as `device_ip`.
 4. Ten independent detectors evaluate that shared context.
-5. The engine deduplicates and prioritizes findings, then attaches capture, packet, stream, and artifact provenance.
+5. The engine deduplicates findings, attributes them to the device under test (`affected_device`, `device_related`), prioritizes them, attaches capture, packet, stream, and artifact provenance, and builds the observation inventory.
 6. Django persists the result and extracted URL/hash artifacts.
 7. The frontend displays findings and can request Markdown, text, or JSON reports.
 8. Optional URL/hash enrichment runs only for artifacts selected by the user.
@@ -31,7 +31,10 @@ IoT PreTester 4.6 Demo analyzes existing `.pcap` and `.pcapng` files and turns p
 ```text
 django_project/analyzer/
 ├── context.py              Normalized capture data model
-├── extract_pyshark.py      PCAP/PCAPNG extraction
+├── extract_tshark.py       PCAP/PCAPNG extraction (tshark -T ek)
+├── tls_tables.py           Cipher-suite and version names generated from tshark
+├── allowlist.py            Expected provider domains (lowers severity, never hides)
+├── inventory.py            Complete observation inventory for the report
 ├── engine.py               Detector orchestration and result assembly
 ├── findings.py             Shared finding taxonomy
 ├── provenance.py           Packet- and stream-level evidence references
@@ -43,7 +46,9 @@ django_project/analyzer/
 └── tests/                  Synthetic unit and regression tests
 ```
 
-The package does not import Django. pyshark and external tools are loaded only at the integration points that need them, so detector tests can run on synthetic contexts without tshark.
+The package does not import Django. tshark and external tools are invoked only at the integration points that need them, so detector tests run on synthetic contexts without tshark; `tests/test_regressions.py` additionally exercises the real extractor against `demo_files/` when tshark is available.
+
+Certificates from TLS 1.2 handshakes are parsed from DER with `cryptography` (subject, issuer, validity, signature algorithm, key size, SAN). TLS 1.3 encrypts the certificate; the inventory states this explicitly.
 
 ## Finding model
 
@@ -51,6 +56,12 @@ Every detector produces the same core fields: ID, title, category, severity, con
 
 Severity measures the potential impact if a finding is true. Confidence measures how strongly the capture supports it. Keeping these dimensions separate prevents low-confidence indicators from being presented as confirmed critical events.
 
+When a device address is configured, every finding carries `evidence.device_related` and, if it touches the device, `affected_device`. Findings involving the device sort before equally severe findings about other hosts; the summary counts both groups.
+
+## Observation inventory
+
+Besides findings, every result contains an `inventory`: capture facts, the device under test with all its peers, internal hosts, external endpoints (with resolved names), observed services, TLS posture per handshake, DNS names, HTTP hosts and file transfers, clear-text flows and service events, and a coverage section (encrypted vs. undecoded flows, truncation, TLS 1.3 certificate visibility). It is rendered in all report formats and in the findings page so that an analyst can verify that nothing was overlooked.
+
 ## Detector set
 
 1. Plaintext credentials
@@ -66,6 +77,11 @@ Severity measures the potential impact if a finding is true. Confidence measures
 
 ## Design decisions
 
+- Parse each capture with tshark's native field export instead of per-packet PDML; a 270 MB capture takes about 15 s instead of 10 minutes, and captures are read completely by default.
+- Retain payload text only from unencrypted flows so secret scanning is never exhausted by ciphertext.
+- Judge TLS by the negotiated ServerHello parameters; what a client merely offers is reported at lower severity.
+- Base clear-text protocol findings on what tshark dissected, not on port numbers alone.
+- Use the provider allowlist to lower severity of expected background traffic, never to suppress it.
 - Parse each capture once and share normalized records across all detectors.
 - Keep detector logic isolated from persistence, presentation, and external tools.
 - Preserve evidence provenance so every finding can be traced back to the capture.
@@ -73,9 +89,14 @@ Severity measures the potential impact if a finding is true. Confidence measures
 - Use a small demo wordlist by default to keep distribution compact and behavior predictable.
 - Store complete analysis results as JSON while keeping extracted artifacts and job status queryable.
 
+## Logging
+
+Django logs to stdout (container log) and to a rotating `backend.log`; gunicorn's error output is routed to stdout as well. A request middleware logs every API call with duration, status, user and an `X-Request-ID`. Each capture analysis writes its own `<capture>.analysis.log` next to the upload (thread-filtered, so concurrent analyses do not mix), records the failure reason in `File.error_message`, and exposes both through the API and the findings page. The frontend keeps a ring buffer of client-side log entries with API timings and uncaught errors.
+
 ## Known demo limitations
 
 - SQLite and in-process background threads are suitable for a controlled demo, not horizontal production scaling.
-- Capture quality and protocol encryption directly limit what can be detected.
+- Capture quality and protocol encryption directly limit what can be detected; TLS 1.3 hides server certificates, and encrypted payloads are never inspected.
+- Provider allowlisting is suffix-based; a device vendor's own cloud domains should be added via `ANALYZER_BENIGN_DOMAINS` to avoid low-severity noise.
 - URL probes observe the endpoint at probe time, which can differ from capture time.
 - The demo does not provide live packet capture, multi-tenant isolation, or production identity-provider integration.

README.md

@@ -5,7 +5,8 @@ IoT PreTester 4.6 Demo is a partner-ready demonstration platform for security an
 ## Demo scope
 
 - Upload and analyze `.pcap` and `.pcapng` captures.
-- Detect credential exposure, secrets, suspicious traffic patterns, weak protocols, TLS issues, DNS anomalies, scanning, and possible exfiltration.
+- Detect credential exposure (FTP, Telnet, POP3, IMAP, SMTP, HTTP forms/JSON/Basic, MQTT, SNMP), secrets, suspicious traffic patterns, clear-text and legacy protocols (incl. MQTT, CoAP, RTSP, Modbus, TFTP), TLS issues (negotiated version/cipher, client capabilities, certificates), DNS anomalies, scanning, brute force, unprotected firmware transfers, and possible exfiltration.
+- Present a complete observation inventory (device peers, hosts, endpoints, services, TLS posture, DNS, coverage) so nothing is overlooked even when no finding is raised.
 - Review affected endpoints, evidence, packet references, severity, confidence, and recommendations.
 - Export reports as Markdown, plain text, or JSON.
 - Optionally run user-selected URL probes and offline hash checks.
@@ -19,7 +20,7 @@ Firmware binary analysis and historical prototype pages are intentionally not pa
 - At least 4 GB of free memory for the two application containers
 - A modern web browser
 
-No local Python, Node.js, tshark, nmap, or hashcat installation is required for the Docker workflow.
+No local Python, Node.js, tshark, nmap, or hashcat installation is required for the Docker workflow. For headless use outside Docker, install tshark (Wireshark 4.0 or newer) and `pip install -r requirements.txt`.
 
 ## Quick start
 
@@ -80,10 +81,26 @@ The analysis core can also run without Django:
 ```bash
 cd django_project
 python -m analyzer.cli ../demo_files/demo_capture.pcapng -f md -o report.md
+python -m analyzer.cli capture.pcapng -f json --device-ip 192.168.1.50 -o report.json
 ```
 
+`--device-ip` marks the device under test: findings that involve it are attributed and sorted first, and the report's observation inventory lists every peer, service, TLS handshake and DNS name of that device. The web UI passes the IP entered at upload time automatically.
+
 Exit code `0` means no high/critical findings, `2` means at least one high/critical finding, and `1` indicates an execution error.
 
+## Logs and troubleshooting
+
+| Where | What |
+| --- | --- |
+| `docker compose logs -f backend` | Application log (requests, analysis phases, errors with tracebacks) |
+| `/app/logs/backend.log` in the backend container | Same log as a rotating file (10 MB × 5) |
+| `/app/files/uploads/<capture>.analysis.log` | Complete log of one capture analysis; shown in the UI when an analysis fails |
+| `GET /api/get_analysis_log/<file_id>` | Tail of that per-capture log |
+| `GET /api/health` | tshark/hashcat/nmap availability, analyzer version, upload directory, log level |
+| Browser console | Client log with API timings; `window.__iotpDumpLogs()` copies the last 300 entries |
+
+Set `LOG_LEVEL=debug` in `.env` to log every detector step and the exact tshark command lines. Every API response carries an `X-Request-ID` header that also appears in the backend log line for that request. In the browser, `localStorage.setItem("iotp_log_level", "debug")` raises the client log level without a rebuild.
+
 ## Demo safety notes
 
 - Only upload captures that you are authorized to inspect. Captures can contain credentials, tokens, personal data, and internal addresses.

django_project/IoTAnalyzer/analyze/security_analysis.py

@@ -5,15 +5,43 @@ from __future__ import annotations
 import json
 import logging
 import os
+import threading
+import time
+import traceback
 
 from django.conf import settings
-from analyzer import AnalysisEngine, extract_urls, extract_hashes
-from analyzer.extract_pyshark import build_context_from_pcap
+from django.db import connection
+from analyzer import AnalysisEngine, extract_urls, extract_hashes, __version__ as analyzer_version
+from analyzer.extract_tshark import build_context_from_pcap
 from analyzer.findings import Severity
+from analyzer import logutil
 from IoTAnalyzer.models import FindingReport, UrlRaw, HashRaw
 
 logger = logging.getLogger(__name__)
 
+
+def analysis_log_path(filename: str) -> str | None:
+    """Per-capture log written next to the upload."""
+    path = resolve_capture_path(filename)
+    if not path:
+        return None
+    return path + ".analysis.log"
+
+
+def _error_summary(exc: BaseException) -> str:
+    """Short, user-facing description of a failure with the failing location."""
+    tb = traceback.extract_tb(exc.__traceback__)
+    where = f" ({tb[-1].filename.rsplit('/', 1)[-1]}:{tb[-1].lineno})" if tb else ""
+    return f"{type(exc).__name__}: {exc}{where}"[:1000]
+
+
+def _set_error(file_id: int, message: str | None) -> None:
+    from IoTAnalyzer.models import File
+    try:
+        File.objects.filter(id=file_id).update(error_message=message)
+    except Exception as exc:
+        logger.warning("Could not store error message for file=%s: %s", file_id, exc)
+
 _CANDIDATE_DIRS = [
     str(settings.CAPTURE_UPLOAD_DIR),
 ]
@@ -109,38 +137,103 @@ def _persist_artifacts(ctx, raw_dataset_id: int) -> None:
         logger.exception("Hash persistence failed: %s", exc)
 
 
-def analyze_capture(filename: str, file_id: int, raw_dataset_id: int | None = None):
-    """Analyze a capture once and persist findings plus extracted artifacts."""
+def analyze_capture(filename: str, file_id: int, raw_dataset_id: int | None = None,
+                    packet_limit: int | None = None, device_ip: str | None = None):
+    """Analyze a capture once and persist findings plus extracted artifacts.
+
+    ``device_ip`` is the address entered at upload time. It focuses the report on
+    the device under test; when omitted it is looked up from the File record.
+    Raises on failure so the caller can record the reason.
+    """
     path = resolve_capture_path(filename)
     if not path:
-        logger.warning("Capture file not found: %s", filename)
-        return None
+        logger.error("Capture file not found: %s (searched: %s)", filename, _CANDIDATE_DIRS)
+        raise FileNotFoundError(f"Capture file {filename} not found in {_CANDIDATE_DIRS}")
 
-    try:
-        ctx = build_context_from_pcap(path)
-        result = AnalysisEngine().analyze_context(ctx)
-    except Exception as exc:
-        logger.exception("Security analysis failed for %s: %s", filename, exc)
-        return None
+    if packet_limit is None:
+        packet_limit = getattr(settings, "CAPTURE_PACKET_LIMIT", 0)
 
-    try:
-        # Reanalysis replaces the previous report atomically at the file level.
-        FindingReport.objects.filter(file_id=file_id).delete()
-        FindingReport.objects.create(
-            file_id=file_id,
-            result_json=json.dumps(result.to_dict()),
-            finding_count=len(result.findings),
-            max_severity=_max_severity(result),
-        )
-        logger.info("Stored finding report: file_id=%s, findings=%s",
-                    file_id, len(result.findings))
-    except Exception as exc:
-        logger.exception("Finding report persistence failed: %s", exc)
+    if device_ip is None:
+        try:
+            from IoTAnalyzer.models import File
+            device_ip = File.objects.filter(id=file_id).values_list("ip_address", flat=True).first()
+        except Exception:
+            device_ip = None
+
+    started = time.time()
+    logger.info("Capture analysis start: file_id=%s path=%s device_ip=%s packet_limit=%s analyzer=%s",
+                file_id, path, device_ip, packet_limit or "unlimited", analyzer_version)
+    ctx = build_context_from_pcap(path, packet_limit=packet_limit or None, device_ip=device_ip)
+    if ctx.truncated:
+        logger.warning("Capture %s analyzed partially: %s", file_id, ctx.truncation_reason)
+    result = AnalysisEngine().analyze_context(ctx)
+
+    # Reanalysis replaces the previous report atomically at the file level.
+    payload = json.dumps(result.to_dict())
+    FindingReport.objects.filter(file_id=file_id).delete()
+    FindingReport.objects.create(
+        file_id=file_id,
+        result_json=payload,
+        finding_count=len(result.findings),
+        max_severity=_max_severity(result),
+    )
+    logger.info("Stored finding report: file_id=%s findings=%s max_severity=%s json=%d bytes",
+                file_id, len(result.findings), _max_severity(result), len(payload))
 
     _persist_artifacts(ctx, raw_dataset_id)
-
+    logger.info("Capture analysis finished: file_id=%s in %.1fs", file_id, time.time() - started)
     return result
 
 
 def run_security_analysis(filename: str, file_id: int):
-    return analyze_capture(filename, file_id, raw_dataset_id=None)
+    try:
+        return analyze_capture(filename, file_id, raw_dataset_id=None)
+    except Exception as exc:
+        logger.exception("Security analysis failed for %s: %s", filename, exc)
+        return None
+
+
+def start_capture_analysis(file_id: int, filename: str, raw_dataset_id: int | None = None,
+                           synchronous: bool = False) -> None:
+    """Analyze a capture off the request thread and keep File.status current.
+
+    A multi-hundred-megabyte capture takes far longer than any sane HTTP
+    timeout, so the upload response must not wait for the analysis.
+    """
+    if synchronous:
+        _run_capture_analysis(file_id, filename, raw_dataset_id, close_conn=False)
+        return
+    thread = threading.Thread(
+        target=_run_capture_analysis,
+        args=(file_id, filename, raw_dataset_id),
+        daemon=True,
+        name=f"capture-analysis-{file_id}",
+    )
+    thread.start()
+
+
+def _run_capture_analysis(file_id: int, filename: str, raw_dataset_id: int | None = None,
+                          close_conn: bool = True) -> None:
+    from IoTAnalyzer.models import File
+
+    status = "analysis_failed"
+    error = None
+    log_path = analysis_log_path(filename)
+    try:
+        with logutil.run_log(log_path):
+            _set_error(file_id, None)
+            result = analyze_capture(filename, file_id, raw_dataset_id)
+            status = "analyzed" if result is not None else "analysis_failed"
+    except Exception as exc:
+        error = _error_summary(exc)
+        logger.exception("Capture analysis failed for file=%s: %s", file_id, error)
+    finally:
+        try:
+            File.objects.filter(id=file_id).update(status=status, error_message=error)
+            logger.info("Capture %s completed with status %s%s", file_id, status,
+                        f" (log: {log_path})" if log_path else "")
+        except Exception as exc:
+            logger.exception("Could not persist status for file=%s: %s", file_id, exc)
+        # Close only the worker thread's connection, never the request connection.
+        if close_conn:
+            connection.close()

django_project/IoTAnalyzer/db_create.py

@@ -84,7 +84,7 @@ def create_capture(
     filename: str,
     ip_address: str,
 ) -> File:
-    """Create capture metadata, analyze the stored file, and update its status."""
+    """Create capture metadata and hand the analysis to a background worker."""
     firmware_version = _required_text(firmware_version, "Firmware version", 100)
     filename = os.path.basename(_required_text(filename, "Capture filename", 255))
     if os.path.splitext(filename)[1].lower() not in CAPTURE_EXTENSIONS:
@@ -113,10 +113,10 @@ def create_capture(
             raise ValidationError("This capture is already registered for the device.")
         raw_dataset = RawDataset.objects.create(file=file_entry)
 
-    from .analyze.security_analysis import analyze_capture
+    from .analyze.security_analysis import start_capture_analysis
 
-    result = analyze_capture(filename, file_entry.id, raw_dataset.id)
-    file_entry.status = "analyzed" if result is not None else "analysis_failed"
-    file_entry.save(update_fields=["status"])
-    logger.info("Capture %s completed with status %s", file_entry.id, file_entry.status)
+    # The analysis reads the whole capture with tshark and can run for many
+    # minutes, so it must not block the upload response.
+    start_capture_analysis(file_entry.id, filename, raw_dataset.id)
+    logger.info("Capture %s registered, analysis queued", file_entry.id)
     return file_entry

django_project/IoTAnalyzer/middleware.py

@@ -0,0 +1,36 @@
+"""Request logging for the API."""
+
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+
+logger = logging.getLogger("IoTAnalyzer.request")
+
+
+class RequestLogMiddleware:
+    """Log one line per API request with duration, status, user and a request id."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        started = time.time()
+        request_id = request.headers.get("X-Request-ID") or uuid.uuid4().hex[:12]
+        request.request_id = request_id
+        try:
+            response = self.get_response(request)
+        except Exception:
+            logger.exception("%s %s failed after %.0f ms [rid=%s]", request.method,
+                             request.get_full_path(), (time.time() - started) * 1000, request_id)
+            raise
+        response["X-Request-ID"] = request_id
+        if request.path.startswith("/api/") and request.path != "/api/health":
+            user = getattr(getattr(request, "user", None), "username", None) or "-"
+            level = logging.WARNING if response.status_code >= 500 else \
+                logging.INFO if response.status_code >= 400 else logging.DEBUG
+            logger.log(level, "%s %s -> %s in %.0f ms user=%s [rid=%s]", request.method,
+                       request.get_full_path(), response.status_code,
+                       (time.time() - started) * 1000, user, request_id)
+        return response

django_project/IoTAnalyzer/migrations/0002_file_error_message.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("IoTAnalyzer", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="file",
+            name="error_message",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

django_project/IoTAnalyzer/models.py

@@ -128,6 +128,8 @@ class File(models.Model):
     filename = models.CharField(max_length=255)
     ip_address = models.GenericIPAddressField()
     status = models.CharField(max_length=50, default="pending")
+    # Last analysis failure, shown in the UI so nobody has to dig through logs first.
+    error_message = models.TextField(null=True, blank=True)
     upload_time = models.DateTimeField(auto_now_add=True)
 
     def to_json(self):
@@ -137,6 +139,7 @@ class File(models.Model):
             "ip_address": self.ip_address,
             "firmware_version": self.firmware.version,
             "status": self.status,
+            "error_message": self.error_message,
             "upload_time": self.upload_time.isoformat(),
         }
 

django_project/IoTAnalyzer/urls.py

@@ -28,5 +28,6 @@ urlpatterns = [
     path("api/get_cloud_results/<int:file_id>", views.CloudResultsView.as_view(), name="cloud_results"),
     path("api/start_analysis/<int:file_id>", views.StartAnalysisView.as_view(), name="start_analysis"),
     path("api/get_analysis_status/<int:file_id>", views.AnalysisStatusView.as_view(), name="analysis_status"),
+    path("api/get_analysis_log/<int:file_id>", views.AnalysisLogView.as_view(), name="analysis_log"),
     path("api/get_report/<int:file_id>/<str:fmt>", views.ReportView.as_view(), name="report"),
 ]

django_project/IoTAnalyzer/views.py

@@ -1,7 +1,11 @@
 """REST endpoints exposed by IoT PreTester 4.6 Demo."""
 
 import json
+import logging
+import os
 import re
+import shutil
+import subprocess
 from pathlib import Path
 from uuid import uuid4
 
@@ -16,7 +20,7 @@ from rest_framework.permissions import AllowAny, IsAdminUser
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from .analyze.security_analysis import collect_cloud_results
+from .analyze.security_analysis import analysis_log_path, collect_cloud_results
 from .db_create import (
     create_capture,
     create_device_type,
@@ -48,16 +52,56 @@ from .user_funcs import (
 )
 
 
+logger = logging.getLogger(__name__)
+
+
 def _validation_error(exc: ValidationError) -> Response:
     messages = getattr(exc, "messages", [str(exc)])
     return Response({"error": messages[0]}, status=status.HTTP_400_BAD_REQUEST)
 
 
+def _tool_version(name: str, args=("-v",)) -> str | None:
+    """First line of a tool's version output, or None when it is missing."""
+    path = shutil.which(name)
+    if not path:
+        return None
+    try:
+        out = subprocess.run([path, *args], capture_output=True, text=True, timeout=15)
+        first = (out.stdout or out.stderr or "").strip().splitlines()
+        return first[0][:120] if first else path
+    except Exception as exc:
+        return f"{path} (version check failed: {exc})"
+
+
 class HealthView(APIView):
+    """Service status including the external tools the analysis depends on."""
+
     permission_classes = [AllowAny]
 
     def get(self, request):
-        return Response({"status": "ok", "service": "IoT PreTester 4.6 Demo"})
+        from analyzer import __version__ as analyzer_version
+
+        tshark = _tool_version(os.environ.get("TSHARK_PATH") or "tshark")
+        upload_dir = str(settings.CAPTURE_UPLOAD_DIR)
+        data = {
+            "status": "ok" if tshark else "degraded",
+            "service": "IoT PreTester 4.6 Demo",
+            "analyzer_version": analyzer_version,
+            "tools": {
+                "tshark": tshark,
+                "hashcat": _tool_version("hashcat", ("--version",)),
+                "nmap": _tool_version("nmap", ("--version",)),
+            },
+            "upload_dir": upload_dir,
+            "upload_dir_writable": os.access(upload_dir, os.W_OK),
+            "log_dir": str(getattr(settings, "LOG_DIR", "")),
+            "log_level": getattr(settings, "LOG_LEVEL", "INFO"),
+            "packet_limit": settings.CAPTURE_PACKET_LIMIT or "unlimited",
+        }
+        if not tshark:
+            data["error"] = "tshark is not installed; capture analysis cannot run."
+            logger.error("Health check: tshark not found on PATH")
+        return Response(data)
 
 
 class LoginView(APIView):
@@ -293,15 +337,30 @@ class DeleteDeviceTypeView(APIView):
         return Response({"status": "success"})
 
 
+# Capture status -> payload shown while no finding report exists yet.
+_PENDING_CAPTURE_STATES = {
+    "analyzing": ("analyzing", "The security analysis is still running."),
+    "pending": ("analyzing", "The security analysis is queued."),
+    "analysis_failed": ("failed", "The security analysis failed. The error and the analysis log are shown below."),
+}
+
+
 class FindingsView(APIView):
     def get(self, request, file_id):
-        get_object_or_404(File, id=file_id)
+        capture = get_object_or_404(File, id=file_id)
         report = FindingReport.objects.filter(file_id=file_id).first()
         if report is None:
-            return Response(
+            state, message = _PENDING_CAPTURE_STATES.get(
+                capture.status,
+                ("empty", "No security analysis is available for this capture."),
+            )
+            response = Response(
                 {
-                    "status": "empty",
-                    "message": "No security analysis is available for this capture.",
+                    "status": state,
+                    "capture_status": capture.status,
+                    "message": message,
+                    "error": capture.error_message,
+                    "log_available": _log_exists(capture),
                     "summary": {
                         "total": 0,
                         "by_severity": {},
@@ -312,7 +371,47 @@ class FindingsView(APIView):
                     "findings": [],
                 }
             )
-        return Response(json.loads(report.result_json))
+            response["Cache-Control"] = "no-store"
+            return response
+        payload = json.loads(report.result_json)
+        payload["status"] = "ready"
+        payload["capture_status"] = capture.status
+        payload["error"] = capture.error_message
+        payload["log_available"] = _log_exists(capture)
+        response = Response(payload)
+        response["Cache-Control"] = "no-store"
+        return response
+
+
+def _log_exists(capture: File) -> bool:
+    path = analysis_log_path(capture.filename)
+    return bool(path and os.path.exists(path))
+
+
+class AnalysisLogView(APIView):
+    """Tail of the per-capture analysis log for troubleshooting."""
+
+    def get(self, request, file_id):
+        from analyzer.logutil import tail
+
+        capture = get_object_or_404(File, id=file_id)
+        path = analysis_log_path(capture.filename)
+        try:
+            lines = max(20, min(int(request.GET.get("lines", 400)), 5000))
+        except (TypeError, ValueError):
+            lines = 400
+        if not path or not os.path.exists(path):
+            return Response(
+                {"file_id": capture.id, "status": capture.status, "error": capture.error_message,
+                 "log": "", "available": False,
+                 "message": "No analysis log exists for this capture yet."},
+            )
+        response = Response({
+            "file_id": capture.id, "status": capture.status, "error": capture.error_message,
+            "available": True, "path": os.path.basename(path), "log": tail(path, lines),
+        })
+        response["Cache-Control"] = "no-store"
+        return response
 
 
 class CloudResultsView(APIView):

django_project/IoTPreTester/settings.py

@@ -47,6 +47,7 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "IoTAnalyzer.middleware.RequestLogMiddleware",
 ]
 
 ROOT_URLCONF = "IoTPreTester.urls"
@@ -118,6 +119,71 @@ STATIC_ROOT = BASE_DIR / "staticfiles"
 CAPTURE_UPLOAD_DIR = Path(
     os.environ.get("CAPTURE_UPLOAD_DIR", BASE_DIR.parent / "uploads")
 )
+
+
+def env_int(name: str, default: int) -> int:
+    try:
+        return int(os.environ.get(name, default))
+    except (TypeError, ValueError):
+        return default
+
+
+# Bounds for a single capture analysis pass. 0 disables the packet limit and
+# reads the capture completely; large captures then take correspondingly longer.
+CAPTURE_PACKET_LIMIT = env_int("CAPTURE_PACKET_LIMIT", 0)
+CAPTURE_MAX_TEXT_PAYLOADS = env_int("CAPTURE_MAX_TEXT_PAYLOADS", 20000)
+CAPTURE_MAX_PAYLOAD_CHARS = env_int("CAPTURE_MAX_PAYLOAD_CHARS", 16384)
+
+# --- Logging -----------------------------------------------------------------
+# Console output goes to the container log; backend.log keeps a rotating copy.
+# Every capture analysis additionally writes <capture>.analysis.log next to
+# the upload, which the API exposes for troubleshooting.
+LOG_DIR = Path(os.environ.get("LOG_DIR", BASE_DIR.parent / "logs"))
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "info").upper()
+try:
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    _FILE_LOGGING = True
+except OSError:
+    _FILE_LOGGING = False
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {
+            "format": "%(asctime)s %(levelname)-7s [%(name)s] %(message)s",
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
+            "stream": "ext://sys.stdout",
+        },
+        **({
+            "file": {
+                "class": "logging.handlers.RotatingFileHandler",
+                "formatter": "standard",
+                "filename": str(LOG_DIR / "backend.log"),
+                "maxBytes": 10 * 1024 * 1024,
+                "backupCount": 5,
+                "encoding": "utf-8",
+            },
+        } if _FILE_LOGGING else {}),
+    },
+    "root": {
+        "handlers": ["console"] + (["file"] if _FILE_LOGGING else []),
+        "level": LOG_LEVEL,
+    },
+    "loggers": {
+        "django": {"level": "INFO", "propagate": True},
+        "django.request": {"level": "WARNING", "propagate": True},
+        "django.server": {"level": "INFO", "propagate": True},
+        "IoTAnalyzer": {"level": LOG_LEVEL, "propagate": True},
+        "analyzer": {"level": LOG_LEVEL, "propagate": True},
+    },
+}
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 APPEND_SLASH = False
 

django_project/analyzer/README.md

@@ -5,7 +5,10 @@
 ## Main modules
 
 - `context.py` defines the normalized `CaptureContext` records.
-- `extract_pyshark.py` builds the context through tshark/pyshark.
+- `extract_tshark.py` builds the context by streaming `tshark -T ek` output (two passes: metadata, then clear-text payloads).
+- `tls_tables.py` maps tshark's numeric cipher-suite and version values to names.
+- `allowlist.py` lists expected provider domains used to lower severity.
+- `inventory.py` builds the complete observation inventory.
 - `detectors/` contains one detection concern per module.
 - `engine.py` runs detectors and assembles the final result.
 - `scoring.py` deduplicates and prioritizes findings.
@@ -21,7 +24,9 @@ From `django_project/`:
 ```bash
 python -m analyzer.cli ../demo_files/demo_capture.pcapng
 python -m analyzer.cli capture.pcapng -f json -o report.json
-python -m analyzer.cli capture.pcapng -f txt --packet-limit 100000
+python -m analyzer.cli capture.pcapng -f txt --device-ip 192.168.1.50
+python -m analyzer.cli capture.pcapng -f json --packet-limit 100000
+python -m analyzer.cli capture.pcapng -v --log-file analysis.log   # debug log on stderr and in a file
 ```
 
 Exit codes are `0` for no high/critical findings, `2` when at least one high/critical finding exists, and `1` for execution errors.
@@ -32,8 +37,9 @@ Exit codes are `0` for no high/critical findings, `2` when at least one high/cri
 from analyzer import AnalysisEngine, render
 from analyzer.context import CaptureContext
 
-result = AnalysisEngine().analyze_pcap("capture.pcapng")
+result = AnalysisEngine().analyze_pcap("capture.pcapng", device_ip="192.168.1.50")
 print(render(result, "md"))
+print(result.inventory["device"])
 
 # Detector logic can also be tested without tshark.
 synthetic_result = AnalysisEngine().analyze_context(CaptureContext())
@@ -42,10 +48,20 @@ synthetic_result = AnalysisEngine().analyze_context(CaptureContext())
 ## Tests
 
 ```bash
-python -m unittest analyzer.tests.test_analyzer -v
+python -m unittest analyzer.tests.test_analyzer analyzer.tests.test_regressions -v
 ```
 
-The unit suite uses synthetic contexts and does not require tshark.
+The unit suite uses synthetic contexts and does not require tshark. `test_regressions.py` also runs the real extractor against `demo_files/demo_capture.pcapng` when tshark is installed (set `ANALYZER_DEMO_DIR` to point at another demo directory).
+
+## Configuration
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `TSHARK_PATH` | `tshark` on PATH | tshark binary used for extraction |
+| `CAPTURE_PACKET_LIMIT` | `0` | Packets to read; `0` reads the whole capture |
+| `CAPTURE_MAX_TEXT_PAYLOADS` | `50000` | Retained clear-text payload entries |
+| `CAPTURE_MAX_PAYLOAD_CHARS` | `16384` | Characters kept per payload entry |
+| `ANALYZER_BENIGN_DOMAINS` | empty | Extra provider domain suffixes for `allowlist.py` |
 
 ## Adding a detector
 

django_project/analyzer/__init__.py

@@ -1,17 +1,7 @@
 """Framework-independent analysis core for IoT PreTester 4.6 Demo.
 
-The package is independent of Django and supports both backend and CLI use.
-The pyshark/tshark extractor is loaded only when a capture is analyzed.
-
-Headless example:
-    from analyzer import AnalysisEngine, render
-    result = AnalysisEngine().analyze_pcap("capture.pcapng")
-    print(render(result, "md"))
-
-Synthetic-context example without tshark:
-    from analyzer import AnalysisEngine
-    from analyzer.context import CaptureContext
-    result = AnalysisEngine().analyze_context(CaptureContext(...))
+Usable from Django and the CLI; tshark is only invoked when a capture is read.
+See README.md for examples.
 """
 
 from __future__ import annotations
@@ -27,7 +17,7 @@ from .artifacts import (
 # Importing the package registers every detector.
 from . import detectors  # noqa: F401
 
-__version__ = "4.6.0-demo.0"
+__version__ = "4.6.2-demo.0"
 
 __all__ = [
     "Finding", "Category", "Severity", "Confidence",

django_project/analyzer/allowlist.py

@@ -0,0 +1,65 @@
+"""Expected provider domains (cloud, CDN, OS, PKI).
+
+Used to lower severity, never to hide an observation. Extend with the
+ANALYZER_BENIGN_DOMAINS environment variable (comma-separated suffixes).
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+BENIGN_SUFFIXES: set[str] = {
+    # Apple
+    "apple.com", "aaplimg.com", "icloud.com", "apple-dns.net", "cdn-apple.com",
+    "mzstatic.com", "itunes-apple.com", "apple.map.fastly.net", "push.apple.com",
+    "ls-apple.com.akadns.net", "edge.apple", "apple-cloudkit.com",
+    # Akamai
+    "akadns.net", "akamai.net", "akamaiedge.net", "akamai.com", "akamaized.net",
+    "akamaihd.net",
+    # AWS
+    "amazonaws.com", "cloudfront.net", "awsstatic.com", "aws.dev", "amazon.com",
+    # Google / Firebase
+    "googleapis.com", "google.com", "gstatic.com", "ggpht.com", "youtube.com",
+    "crashlytics.com", "firebaseio.com", "firebase.com", "app-measurement.com",
+    "googleusercontent.com", "google-analytics.com", "doubleclick.net", "1e100.net",
+    "googlevideo.com", "gvt1.com", "gvt2.com",
+    # Apple analytics endpoint used by many apps
+    "app-analytics-services.com",
+    # CDNs
+    "fastly.net", "fbcdn.net", "cloudflare.com", "cloudflare.net",
+    "cloudflare-dns.com", "edgekey.net", "edgesuite.net", "llnwd.net", "cdn77.org",
+    "azureedge.net", "aliyuncs.com", "alicdn.com",
+    # Microsoft
+    "microsoft.com", "windows.net", "azure.com", "windowsupdate.com", "office.com",
+    "msftncsi.com", "live.com", "msftconnecttest.com", "office365.com",
+    # PKI / time
+    "digicert.com", "sectigo.com", "letsencrypt.org", "globalsign.com", "godaddy.com",
+    "ntp.org", "pool.ntp.org", "time.apple.com", "time.google.com", "time.windows.com",
+    # Reverse-DNS / special-use
+    "in-addr.arpa", "ip6.arpa", "resolver.arpa", "arpa", "local", "home.arpa",
+}
+
+
+def _load_extra() -> set[str]:
+    raw = os.environ.get("ANALYZER_BENIGN_DOMAINS", "")
+    return {s.strip().lower().lstrip(".") for s in raw.split(",") if s.strip()}
+
+
+_EXTRA = _load_extra()
+
+
+def is_benign(name: Optional[str]) -> bool:
+    """Whether a host name belongs to an expected provider."""
+    if not name:
+        return False
+    n = str(name).rstrip(".").lower()
+    return any(n == s or n.endswith("." + s) for s in BENIGN_SUFFIXES | _EXTRA)
+
+
+def benign_name_for_ip(ctx, ip: Optional[str]) -> Optional[str]:
+    """Return a benign provider name that resolved to the address, if any."""
+    for name in sorted(ctx.names_for_ip(ip)):
+        if is_benign(name):
+            return name
+    return None

django_project/analyzer/artifacts.py

@@ -1,6 +1,6 @@
 """Extract endpoint and crackable hash artifacts from a capture context.
 
-Extraction is independent of Django, pyshark, and I/O. Active endpoint probing
+Extraction is independent of Django, tshark, and I/O. Active endpoint probing
 and hash cracking remain isolated in ``analyzer.enrich``.
 """
 
@@ -143,6 +143,38 @@ _URL_RE = re.compile(
 )
 
 
+# Endpoints are only trustworthy from cleartext application data. Scanning
+# encrypted/binary payloads (TLS, QUIC, SSH, raw TCP) with a loose domain regex
+# invents host-shaped noise from random bytes, so payload extraction is limited
+# to protocols whose bytes are actually text.
+_CLEARTEXT_PROTOS = {
+    "http", "http2", "telnet", "ftp", "ftp-data", "ftp_data", "smtp", "pop",
+    "pop3", "imap", "snmp", "tftp", "ldap", "sip", "irc", "syslog", "rtsp",
+    "xmpp", "mqtt", "urlencoded-form", "json", "xml", "data-text-lines",
+}
+
+_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+
+
+def _looks_like_host(value: str) -> bool:
+    """Reject byte noise that the URL regex mistakes for a bare domain."""
+    v = value.split("://")[-1].split("/")[0]
+    if _IPV4_RE.match(v):
+        return all(0 <= int(o) <= 255 for o in v.split("."))
+    if len(v) < 4 or ".." in v:
+        return False
+    labels = v.split(".")
+    if len(labels) < 2:
+        return False
+    tld = labels[-1]
+    if not tld.isalpha() or not (2 <= len(tld) <= 24):
+        return False
+    # At least one substantive label — kills noise like "4v.b.di" / "q.p.fh".
+    if not any(len(lbl) >= 3 for lbl in labels[:-1]):
+        return False
+    return True
+
+
 def _normalize_host(value: Optional[str]) -> Optional[str]:
     if not value:
         return None
@@ -187,8 +219,12 @@ def extract_urls(ctx: CaptureContext) -> list[UrlArtifact]:
                 art.observed_ports.add(r.dst_port)
 
     for p in ctx.text_payloads:
+        if (p.get("protocol") or "").lower() not in _CLEARTEXT_PROTOS:
+            continue
         text = p.get("text") or ""
         for m in _URL_RE.findall(text):
+            if not _looks_like_host(m):
+                continue
             add(m, "payload", src_ip=p.get("src_ip"), dst_ip=p.get("dst_ip"),
                 timestamp=p.get("timestamp"), packet_number=p.get("packet_number"))
 
@@ -226,7 +262,11 @@ def extract_hashes(ctx: CaptureContext) -> list[HashArtifact]:
                 timestamp=am.timestamp, packet_number=am.packet_number)
 
     # Raw 32–64 character hex values use a mode only when the length is known.
+    # Only cleartext payloads are scanned; TLS/QUIC ciphertext is full of 32–64
+    # char hex runs that are not hashes.
     for p in ctx.text_payloads:
+        if (p.get("protocol") or "").lower() not in _CLEARTEXT_PROTOS:
+            continue
         text = p.get("text") or ""
         for m in _HASH_HEX_RE.findall(text):
             mode, fmt = identify_hash_mode(m)

django_project/analyzer/cli.py

@@ -1,25 +1,19 @@
 """Headless CLI for IoT PreTester 4.6 Demo.
 
-Examples:
-    python -m analyzer.cli capture.pcapng
-    python -m analyzer.cli capture.pcapng --format json -o report.json
-    python -m analyzer.cli capture.pcapng --format txt --packet-limit 50000
-
-Exit codes:
-    0  Analysis completed without high or critical findings
-    2  Analysis completed with at least one high or critical finding
-    1  Capture or runtime dependency error
+Exit codes: 0 no high/critical findings, 2 at least one, 1 runtime error.
 """
 
 from __future__ import annotations
 
 import argparse
+import logging
 import sys
 
 from . import __version__
 from .engine import AnalysisEngine
 from .findings import Severity
 from .reporting import render, FORMATS
+from . import logutil
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -34,6 +28,10 @@ def build_parser() -> argparse.ArgumentParser:
                    help="Output file (default: standard output)")
     p.add_argument("--packet-limit", type=int, default=None,
                    help="Maximum packets to read from a large capture")
+    p.add_argument("--device-ip", default=None,
+                   help="IP address of the device under test (focuses and attributes findings)")
+    p.add_argument("-v", "--verbose", action="store_true", help="Debug logging on stderr")
+    p.add_argument("--log-file", default=None, help="Write the analysis log to this file")
     p.add_argument(
         "--version",
         action="version",
@@ -44,15 +42,22 @@ def build_parser() -> argparse.ArgumentParser:
 
 def main(argv=None) -> int:
     args = build_parser().parse_args(argv)
+    logutil.configure_console("DEBUG" if args.verbose else "WARNING")
 
     try:
-        result = AnalysisEngine().analyze_pcap(args.capture, packet_limit=args.packet_limit)
+        with logutil.run_log(args.log_file):
+            result = AnalysisEngine().analyze_pcap(args.capture, packet_limit=args.packet_limit,
+                                                   device_ip=args.device_ip)
     except FileNotFoundError as exc:
         print(f"Error: {exc}", file=sys.stderr)
         return 1
     except RuntimeError as exc:
         print(f"Error: {exc}", file=sys.stderr)
         return 1
+    except Exception as exc:
+        logging.getLogger("analyzer.cli").exception("Unexpected failure")
+        print(f"Error: {exc!r}", file=sys.stderr)
+        return 1
 
     report = render(result, args.format)
 

django_project/analyzer/context.py

@@ -1,7 +1,7 @@
 """Normalized capture data consumed by every detector.
 
-The module is independent of Django and pyshark so detector logic can be tested
-with synthetic data. ``extract_pyshark.py`` performs the actual PCAP extraction.
+The module is independent of Django and tshark so detector logic can be tested
+with synthetic data. ``extract_tshark.py`` performs the actual PCAP extraction.
 """
 
 from __future__ import annotations
@@ -19,16 +19,28 @@ from typing import Optional
 class Host:
     ip: str
     macs: set[str] = field(default_factory=set)
-    hostnames: set[str] = field(default_factory=set)
+    hostnames: set[str] = field(default_factory=set)  # DHCP, mDNS, reverse DNS names
     bytes_sent: int = 0
     bytes_received: int = 0
-    ports_offered: set[int] = field(default_factory=set)  # Ports on which the host responds
-    ports_contacted: set[int] = field(default_factory=set)  # Ports contacted by the host
+    packets_sent: int = 0
+    packets_received: int = 0
+    first_seen: Optional[float] = None
+    last_seen: Optional[float] = None
+    ports_offered: set[int] = field(default_factory=set)   # Ports the host answered on (server role)
+    ports_contacted: set[int] = field(default_factory=set)  # Destination ports the host contacted
+    dhcp_vendor_class: Optional[str] = None
+    user_agents: set[str] = field(default_factory=set)
+    # Application protocols seen with this host as an endpoint (from frame.protocols).
+    app_protocols: set[str] = field(default_factory=set)
 
     @property
     def is_private(self) -> bool:
         return _is_private(self.ip)
 
+    @property
+    def packets(self) -> int:
+        return self.packets_sent + self.packets_received
+
 
 @dataclass
 class Conversation:
@@ -43,23 +55,56 @@ class Conversation:
     bytes: int = 0
     bytes_src_to_dst: int = 0
     bytes_dst_to_src: int = 0
+    packets_src_to_dst: int = 0
+    packets_dst_to_src: int = 0
+    payload_bytes: int = 0  # Application payload only (no headers)
     first_seen: Optional[float] = None
     last_seen: Optional[float] = None
     stream_id: Optional[int] = None
     timestamps: list[float] = field(default_factory=list)  # Used to detect beaconing
     packet_numbers: list[int] = field(default_factory=list)  # Bounded evidence packet list
+    # TCP state summary.
+    syn_count: int = 0        # SYNs sent by src (connection attempts)
+    synack_count: int = 0     # SYN/ACKs sent back by dst (service answered)
+    rst_count: int = 0
+    fin_count: int = 0
+    retransmissions: int = 0
+    # Application protocols dissected on this flow, e.g. {"tls", "http"}.
+    app_protocols: set[str] = field(default_factory=set)
+
+    @property
+    def answered(self) -> bool:
+        """Whether the destination visibly responded (TCP handshake or return traffic)."""
+        if self.protocol == "tcp":
+            return self.synack_count > 0 or self.packets_dst_to_src > 0
+        return self.packets_dst_to_src > 0
+
+    @property
+    def is_encrypted(self) -> bool:
+        return bool(self.app_protocols & ENCRYPTED_PROTOCOLS)
+
+
+# Protocols whose payload is ciphertext and must never be scanned as text.
+ENCRYPTED_PROTOCOLS = {
+    "tls", "ssl", "dtls", "quic", "ssh", "esp", "ipsec", "wireguard", "openvpn",
+    "ikev2", "isakmp", "kerberos", "ntlmssp",
+}
 
 
 @dataclass
 class DnsQuery:
     query_name: str
     qtype: Optional[str] = None
+    is_response: bool = False
     response_ips: list[str] = field(default_factory=list)
+    cnames: list[str] = field(default_factory=list)
+    txt: list[str] = field(default_factory=list)
     rcode: Optional[str] = None  # For example, "NOERROR" or "NXDOMAIN"
     timestamp: Optional[float] = None
     src_ip: Optional[str] = None
     dst_ip: Optional[str] = None
     resolver: Optional[str] = None
+    transport: Optional[str] = None  # "dns", "mdns", "llmnr", "nbns"
     packet_number: Optional[int] = None
 
 
@@ -73,6 +118,7 @@ class HttpRequest:
     cookies: Optional[str] = None
     headers: dict[str, str] = field(default_factory=dict)
     body: Optional[str] = None
+    content_type: Optional[str] = None
     request_bytes: int = 0
     is_tls: bool = False  # True when only TLS metadata is visible
     src_ip: Optional[str] = None
@@ -88,8 +134,14 @@ class HttpResponse:
     status_code: Optional[int] = None
     content_type: Optional[str] = None
     server: Optional[str] = None
+    content_disposition: Optional[str] = None
+    content_length: Optional[int] = None
+    set_cookie: list[str] = field(default_factory=list)
+    www_authenticate: Optional[str] = None
+    request_uri: Optional[str] = None
     src_ip: Optional[str] = None
     dst_ip: Optional[str] = None
+    src_port: Optional[int] = None
     timestamp: Optional[float] = None
     stream_id: Optional[int] = None
     packet_number: Optional[int] = None
@@ -97,20 +149,39 @@ class HttpResponse:
 
 @dataclass
 class TlsSession:
+    """One TLS handshake, aggregated per TCP stream (or QUIC connection)."""
+
     src_ip: Optional[str] = None
     dst_ip: Optional[str] = None
     dst_port: Optional[int] = 443
-    version: Optional[str] = None  # For example, "TLS 1.2" or "TLS 1.0"
-    cipher: Optional[str] = None  # Cipher-suite name
+    src_port: Optional[int] = None
+    transport: str = "tcp"  # "tcp" or "quic"
+    version: Optional[str] = None  # Negotiated version from the ServerHello
+    cipher: Optional[str] = None  # Negotiated cipher-suite name from the ServerHello
     sni: Optional[str] = None
+    alpn: list[str] = field(default_factory=list)
+    client_version: Optional[str] = None  # Highest version the client offered
+    client_offered_versions: list[str] = field(default_factory=list)
+    client_offered_ciphers: list[str] = field(default_factory=list)
+    client_hello_seen: bool = False
+    server_hello_seen: bool = False
     cert_subject_cn: Optional[str] = None
     cert_issuer_cn: Optional[str] = None
+    cert_san: list[str] = field(default_factory=list)
+    cert_not_before: Optional[str] = None
     cert_not_after: Optional[str] = None
     cert_self_signed: Optional[bool] = None
     cert_expired: Optional[bool] = None
+    cert_sig_algorithm: Optional[str] = None
+    cert_key_type: Optional[str] = None
+    cert_key_bits: Optional[int] = None
+    cert_seen: bool = False
+    client_cert_seen: bool = False  # Client presented a certificate (mutual TLS)
+    alerts: list[str] = field(default_factory=list)
     timestamp: Optional[float] = None
     stream_id: Optional[int] = None
     packet_number: Optional[int] = None
+    packet_numbers: list[int] = field(default_factory=list)
 
 
 @dataclass
@@ -147,6 +218,52 @@ class ArpEvent:
     packet_number: Optional[int] = None
 
 
+@dataclass
+class DhcpEvent:
+    message_type: Optional[str] = None  # "discover", "offer", "request", "ack", ...
+    client_mac: Optional[str] = None
+    client_ip: Optional[str] = None     # Assigned/requested address when known
+    hostname: Optional[str] = None
+    vendor_class: Optional[str] = None
+    server_id: Optional[str] = None
+    src_ip: Optional[str] = None
+    timestamp: Optional[float] = None
+    packet_number: Optional[int] = None
+
+
+@dataclass
+class MqttEvent:
+    """MQTT control packet visible in clear text (port 1883 or any unencrypted flow)."""
+
+    msg_type: Optional[str] = None      # "connect", "connack", "publish", "subscribe", ...
+    client_id: Optional[str] = None
+    username: Optional[str] = None
+    password: Optional[str] = None
+    topic: Optional[str] = None
+    return_code: Optional[int] = None   # CONNACK return code
+    src_ip: Optional[str] = None
+    dst_ip: Optional[str] = None
+    dst_port: Optional[int] = None
+    timestamp: Optional[float] = None
+    stream_id: Optional[int] = None
+    packet_number: Optional[int] = None
+
+
+@dataclass
+class ServiceEvent:
+    """Generic clear-text protocol observation (SNMP community, SSH banner, ...)."""
+
+    protocol: str
+    kind: str                       # "snmp_community", "ssh_banner", "coap_request", ...
+    value: Optional[str] = None
+    detail: Optional[str] = None
+    src_ip: Optional[str] = None
+    dst_ip: Optional[str] = None
+    dst_port: Optional[int] = None
+    timestamp: Optional[float] = None
+    packet_number: Optional[int] = None
+
+
 @dataclass
 class FileTransfer:
     filename: Optional[str] = None
@@ -157,6 +274,8 @@ class FileTransfer:
     src_ip: Optional[str] = None
     dst_ip: Optional[str] = None
     direction: Optional[str] = None             # "download" / "upload"
+    uri: Optional[str] = None
+    host: Optional[str] = None
     timestamp: Optional[float] = None
     stream_id: Optional[int] = None
     packet_number: Optional[int] = None
@@ -172,6 +291,23 @@ class PlaintextAuth:
     src_ip: Optional[str] = None
     dst_ip: Optional[str] = None
     dst_port: Optional[int] = None
+    # "success", "failure" or None when the server response was not observed.
+    outcome: Optional[str] = None
+    timestamp: Optional[float] = None
+    stream_id: Optional[int] = None
+    packet_number: Optional[int] = None
+
+
+@dataclass
+class AuthAttempt:
+    """A single authentication attempt result used for brute-force detection."""
+
+    protocol: str
+    success: Optional[bool]
+    src_ip: Optional[str] = None   # Client
+    dst_ip: Optional[str] = None   # Server
+    dst_port: Optional[int] = None
+    username: Optional[str] = None
     timestamp: Optional[float] = None
     stream_id: Optional[int] = None
     packet_number: Optional[int] = None
@@ -187,8 +323,19 @@ class CaptureContext:
 
     path: Optional[str] = None
     packet_count: int = 0
+    # Set when limits stopped the extraction pass before the capture was fully read.
+    truncated: bool = False
+    truncation_reason: Optional[str] = None
+    truncation_reasons: list[str] = field(default_factory=list)
+    # Non-fatal coverage notes (protocols not dissected, undecoded ports, ...).
+    coverage_notes: list[str] = field(default_factory=list)
     start_time: Optional[float] = None
     end_time: Optional[float] = None
+    extractor: Optional[str] = None  # e.g. "tshark 4.2.2"
+
+    # The device under test. When set, findings are attributed to it.
+    device_ip: Optional[str] = None
+    device_macs: set[str] = field(default_factory=set)
 
     hosts: dict[str, Host] = field(default_factory=dict)
     conversations: list[Conversation] = field(default_factory=list)
@@ -198,11 +345,19 @@ class CaptureContext:
     tls_sessions: list[TlsSession] = field(default_factory=list)
     auth_material: list[AuthMaterial] = field(default_factory=list)
     arp_events: list[ArpEvent] = field(default_factory=list)
+    dhcp_events: list[DhcpEvent] = field(default_factory=list)
+    mqtt_events: list[MqttEvent] = field(default_factory=list)
+    service_events: list[ServiceEvent] = field(default_factory=list)
     file_transfers: list[FileTransfer] = field(default_factory=list)
     plaintext_auth: list[PlaintextAuth] = field(default_factory=list)
+    auth_attempts: list[AuthAttempt] = field(default_factory=list)
 
-    # protocol_stats: protocol-name -> packet count
+    # protocol_stats: highest-layer protocol name -> packet count (legacy, kept for detectors)
     protocol_stats: dict[str, int] = field(default_factory=dict)
+    # app_protocol_stats: every protocol in frame.protocols -> packet count
+    app_protocol_stats: dict[str, int] = field(default_factory=dict)
+    icmp_stats: dict[str, int] = field(default_factory=dict)
+    vlan_ids: set[int] = field(default_factory=set)
     # Generic text payloads with packet metadata for secret detection.
     text_payloads: list[dict] = field(default_factory=list)
 
@@ -225,14 +380,50 @@ class CaptureContext:
             return False
         return not _is_private(ip)
 
+    def is_device(self, ip: Optional[str]) -> bool:
+        """Whether the address belongs to the device under test."""
+        return bool(ip) and self.device_ip is not None and ip == self.device_ip
+
+    def involves_device(self, *ips: Optional[str]) -> Optional[bool]:
+        """True/False when a device is configured, None when no device is known."""
+        if self.device_ip is None:
+            return None
+        return any(self.is_device(ip) for ip in ips)
+
     def protocol_count(self, *names: str) -> int:
-        """Return a case-insensitive packet total for the requested protocols."""
+        """Return a case-insensitive packet total for the requested protocols.
+
+        Both the highest-layer statistics and the complete protocol chain are
+        consulted so that e.g. "smb" is found even when a higher layer sits on top.
+        """
         wanted = {n.lower() for n in names}
+        total = sum(
+            count for proto, count in self.app_protocol_stats.items()
+            if str(proto).lower() in wanted
+        )
+        if total:
+            return total
         return sum(
             count for proto, count in self.protocol_stats.items()
             if str(proto).lower() in wanted
         )
 
+    def names_for_ip(self, ip: Optional[str]) -> set[str]:
+        """DNS names that resolved to an address, plus SNI/Host names used with it."""
+        names: set[str] = set()
+        if not ip:
+            return names
+        for q in self.dns_queries:
+            if ip in q.response_ips and q.query_name:
+                names.add(q.query_name.rstrip(".").lower())
+        for t in self.tls_sessions:
+            if t.dst_ip == ip and t.sni:
+                names.add(t.sni.lower())
+        for r in self.http_requests:
+            if r.dst_ip == ip and r.host:
+                names.add(str(r.host).split(":")[0].lower())
+        return names
+
 
 def _is_private(ip: str) -> bool:
     try:
@@ -247,3 +438,16 @@ def _is_private(ip: str) -> bool:
         or addr.is_reserved
         or addr.is_unspecified
     )
+
+
+def is_special(ip: Optional[str]) -> bool:
+    """Broadcast, multicast, unspecified and link-scope addresses that never identify one host."""
+    if not ip:
+        return True
+    if ip in ("0.0.0.0", "255.255.255.255", "::"):
+        return True
+    try:
+        addr = ipaddress.ip_address(ip)
+    except ValueError:
+        return True
+    return addr.is_multicast or addr.is_unspecified or addr.is_loopback

django_project/analyzer/detectors/bruteforce_scan_filetransfer.py

@@ -1,4 +1,4 @@
-"""Detect brute force, scanning, and suspicious file transfers."""
+"""Detect scanning, repeated authentication failures, and risky file transfers."""
 
 from __future__ import annotations
 
@@ -10,145 +10,240 @@ from ..context import CaptureContext
 from ..findings import Finding, Category, Severity, Confidence
 from .base import BaseDetector, register
 
-# File extensions that warrant closer inspection.
-_SUSPICIOUS_EXT = {
-    ".exe", ".dll", ".ps1", ".vbs", ".js", ".jar", ".bat", ".cmd", ".scr",
-    ".hta", ".lnk", ".docm", ".xlsm", ".pptm", ".zip", ".rar", ".7z", ".iso", ".elf",
+# Executable or script content that warrants closer inspection.
+_EXECUTABLE_EXT = {
+    ".exe", ".dll", ".ps1", ".vbs", ".jar", ".bat", ".cmd", ".scr", ".hta", ".lnk",
+    ".docm", ".xlsm", ".pptm", ".msi", ".apk", ".elf", ".sh", ".py", ".pl",
 }
+# Firmware and update images: integrity matters more than malware here.
+_FIRMWARE_EXT = {
+    ".bin", ".img", ".fw", ".ota", ".upd", ".hex", ".rom", ".swu", ".ubi", ".squashfs",
+    ".pkg", ".tgz", ".tar", ".gz", ".xz", ".bz2", ".zip", ".7z", ".rar", ".iso",
+}
+_EXECUTABLE_CT = ("x-msdownload", "x-dosexec", "x-executable", "portable-executable", "x-elf",
+                  "x-msi", "vnd.android.package-archive", "x-sh")
 _PORTSCAN_PORTS = 15
+_PORTSCAN_UNANSWERED = 8
 _HOSTSWEEP_HOSTS = 15
 _FAILED_LOGINS = 5
+_HTTP_401_MIN = 8
 
 
 @register
 class BruteforceScanFileTransferDetector(BaseDetector):
     name = "BruteforceScanFileTransferDetector"
-    description = "Detects scans, repeated authentication failures, and risky downloads."
+    description = "Detects scans, repeated authentication failures, and risky or unprotected file transfers."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
+        findings.extend(self._scans(ctx))
+        findings.extend(self._auth_failures(ctx))
+        findings.extend(self._files(ctx))
+        return self._tag(findings)
 
+    # ---- scanning --------------------------------------------------------- #
+
+    def _scans(self, ctx: CaptureContext) -> list[Finding]:
+        findings: list[Finding] = []
         ports_per_pair: dict[tuple, set[int]] = defaultdict(set)
-        hosts_per_src: dict[str, set[str]] = defaultdict(set)
+        unanswered_per_pair: dict[tuple, set[int]] = defaultdict(set)
         packets_per_pair: dict[tuple, list] = defaultdict(list)
+        hosts_per_src: dict[str, set[str]] = defaultdict(set)
+        unanswered_hosts_per_src: dict[str, set[str]] = defaultdict(set)
         packets_per_src: dict[str, list] = defaultdict(list)
+
         for conv in ctx.conversations:
-            if conv.src_ip and conv.dst_ip:
-                if conv.dst_port:
-                    ports_per_pair[(conv.src_ip, conv.dst_ip)].add(conv.dst_port)
+            if not (conv.src_ip and conv.dst_ip and conv.dst_port):
+                continue
+            pair = (conv.src_ip, conv.dst_ip)
+            ports_per_pair[pair].add(conv.dst_port)
+            if conv.protocol == "tcp" and conv.syn_count and not conv.answered:
+                unanswered_per_pair[pair].add(conv.dst_port)
+            if conv.packet_numbers:
+                packets_per_pair[pair].append(conv.packet_numbers[0])
+                packets_per_src[conv.src_ip].append(conv.packet_numbers[0])
+            # A host sweep scans the *local* network; fan-out to cloud endpoints is normal.
+            if not ctx.is_external(conv.dst_ip) and conv.dst_ip != conv.src_ip:
                 hosts_per_src[conv.src_ip].add(conv.dst_ip)
-                if conv.packet_numbers:
-                    packets_per_pair[(conv.src_ip, conv.dst_ip)].append(conv.packet_numbers[0])
-                    packets_per_src[conv.src_ip].append(conv.packet_numbers[0])
+                if not conv.answered:
+                    unanswered_hosts_per_src[conv.src_ip].add(conv.dst_ip)
 
         for (src, dst), ports in ports_per_pair.items():
-            if len(ports) >= _PORTSCAN_PORTS:
+            unanswered = unanswered_per_pair.get((src, dst), set())
+            if len(ports) >= _PORTSCAN_PORTS or len(unanswered) >= _PORTSCAN_UNANSWERED:
+                conf = Confidence.HIGH if len(unanswered) >= _PORTSCAN_UNANSWERED else Confidence.MEDIUM
                 findings.append(Finding(
                     title="Port scan detected",
                     category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
-                    severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
-                    source_ip=src, destination_ip=dst,
+                    severity=Severity.MEDIUM, confidence=conf,
+                    source_ip=src, destination_ip=dst, protocol="tcp",
                     risk_description=(
-                        "One host contacted many ports on a target, which is characteristic "
-                        "of network reconnaissance."
+                        "One host contacted many ports on a target, most of them without an answer, "
+                        "which is characteristic of network reconnaissance."
                     ),
-                    recommendation="Review the source and block unauthorized external scanners.",
-                    evidence={"distinct_ports": len(ports), "dedup_key": f"portscan:{src}:{dst}"},
+                    recommendation="Review the source and block unauthorized scanners.",
+                    evidence={"distinct_ports": len(ports), "unanswered_ports": len(unanswered),
+                              "sample_ports": sorted(ports)[:12], "target_is_device": ctx.is_device(dst),
+                              "dedup_key": f"portscan:{src}:{dst}"},
                     related_packets=sorted(set(packets_per_pair.get((src, dst), [])))[:8],
                 ))
 
         for src, dsts in hosts_per_src.items():
             if len(dsts) >= _HOSTSWEEP_HOSTS:
+                unanswered = unanswered_hosts_per_src.get(src, set())
+                if len(unanswered) < len(dsts) * 0.5:
+                    continue  # Talking to many answering hosts is normal for a gateway or controller.
                 findings.append(Finding(
                     title="Host sweep detected",
                     category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
-                    severity=Severity.MEDIUM, confidence=Confidence.LOW,
+                    severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
                     source_ip=src,
                     risk_description=(
-                        "One host contacted many different targets, which may indicate a "
-                        "network sweep or worm behavior."
+                        "One host probed many internal addresses, most of which did not answer, "
+                        "which indicates a network sweep or worm-like behaviour."
                     ),
-                    recommendation="Review the source and investigate internal hosts for compromise.",
-                    evidence={"distinct_hosts": len(dsts), "dedup_key": f"sweep:{src}"},
+                    recommendation="Review the source and investigate it for compromise.",
+                    evidence={"distinct_hosts": len(dsts), "unanswered_hosts": len(unanswered),
+                              "dedup_key": f"sweep:{src}"},
                     related_packets=sorted(set(packets_per_src.get(src, [])))[:8],
                 ))
 
-        # Repeated HTTP authorization failures are a brute-force indicator.
-        http_401: dict[tuple, int] = defaultdict(int)
-        http_401_pkts: dict[tuple, list] = defaultdict(list)
+        echo = ctx.icmp_stats.get("echo_request", 0)
+        if echo >= 50:
+            findings.append(Finding(
+                title="High number of ICMP echo requests",
+                category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
+                severity=Severity.LOW, confidence=Confidence.LOW, protocol="icmp",
+                risk_description="Many ICMP echo requests can indicate a ping sweep or a monitoring probe.",
+                recommendation="Verify the source of the echo requests.",
+                evidence={"echo_requests": echo, "dedup_key": "icmp_sweep"},
+            ))
+        return findings
+
+    # ---- authentication failures ----------------------------------------- #
+
+    def _auth_failures(self, ctx: CaptureContext) -> list[Finding]:
+        findings: list[Finding] = []
+        failures: dict[tuple, list] = defaultdict(list)
+        for a in ctx.auth_attempts:
+            if a.success is False:
+                failures[(a.protocol, a.src_ip, a.dst_ip)].append(a)
+        for (proto, client, server), attempts in failures.items():
+            if len(attempts) < _FAILED_LOGINS:
+                continue
+            users = sorted({a.username for a in attempts if a.username})
+            findings.append(Finding(
+                title=f"Repeated failed {proto.upper()} logins",
+                category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
+                severity=Severity.HIGH, confidence=Confidence.HIGH,
+                source_ip=client, destination_ip=server, protocol=proto, port=attempts[0].dst_port,
+                timestamp_first_seen=attempts[0].timestamp, timestamp_last_seen=attempts[-1].timestamp,
+                risk_description=(
+                    f"{len(attempts)} rejected {proto.upper()} login attempts from one client indicate "
+                    "password guessing."
+                ),
+                recommendation="Enable rate limiting or lockout controls, review the source, and enforce strong credentials.",
+                evidence={"failed_attempts": len(attempts), "usernames": users[:10],
+                          "target_is_device": ctx.is_device(server),
+                          "dedup_key": f"bruteforce:{proto}:{client}:{server}"},
+                related_packets=sorted({a.packet_number for a in attempts if a.packet_number})[:8],
+            ))
+
+        # HTTP: only 401 counts (403 is routine for blocked resources).
+        http_401: dict[tuple, list] = defaultdict(list)
         for resp in ctx.http_responses:
-            if resp.status_code in (401, 403):
-                http_401[(resp.dst_ip, resp.src_ip)] += 1
-                if resp.packet_number:
-                    http_401_pkts[(resp.dst_ip, resp.src_ip)].append(resp.packet_number)
-        for (client, server), count in http_401.items():
-            if count >= _FAILED_LOGINS:
+            if resp.status_code == 401:
+                http_401[(resp.dst_ip, resp.src_ip)].append(resp)
+        for (client, server), resps in http_401.items():
+            if len(resps) >= _HTTP_401_MIN:
                 findings.append(Finding(
                     title="Repeated failed HTTP authentication",
                     category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
                     severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
                     source_ip=client, destination_ip=server, protocol="http",
-                    risk_description=(
-                        "Many HTTP 401 or 403 responses may indicate a password guessing attack."
-                    ),
+                    timestamp_first_seen=resps[0].timestamp, timestamp_last_seen=resps[-1].timestamp,
+                    risk_description="Many HTTP 401 responses to one client may indicate password guessing.",
                     recommendation="Enable rate limiting or lockout controls and review the source.",
-                    evidence={"failed_responses": count, "dedup_key": f"bruteforce:{client}:{server}"},
-                    related_packets=sorted(set(http_401_pkts.get((client, server), [])))[:8],
+                    evidence={"failed_responses": len(resps),
+                              "uris": sorted({r.request_uri for r in resps if r.request_uri})[:5],
+                              "target_is_device": ctx.is_device(server),
+                              "dedup_key": f"bruteforce:http:{client}:{server}"},
+                    related_packets=sorted({r.packet_number for r in resps if r.packet_number})[:8],
                 ))
+        return findings
 
-        seen_files: set[str] = set()
-        for req in ctx.http_requests:
-            uri = (req.uri or "").split("?")[0]
-            ext = os.path.splitext(uri)[1].lower()
-            if ext in _SUSPICIOUS_EXT:
-                key = f"file:{req.host}:{uri}"
-                if key in seen_files:
-                    continue
-                seen_files.add(key)
-                findings.append(Finding(
-                    title=f"Potentially harmful file download ({ext})",
-                    category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
-                    severity=Severity.HIGH, confidence=Confidence.MEDIUM,
-                    source_ip=req.src_ip, destination_ip=req.dst_ip, protocol="http", port=req.dst_port,
-                    timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
-                    risk_description=(
-                        "An executable or script-capable file was transferred over HTTP and "
-                        "may represent a malware download."
-                    ),
-                    recommendation="Analyze the file in a sandbox and check its hash against threat intelligence.",
-                    evidence={"host": req.host, "uri": uri, "extension": ext, "dedup_key": key},
-                    related_packets=[p for p in [req.packet_number] if p],
-                ))
+    # ---- file transfers --------------------------------------------------- #
+
+    def _files(self, ctx: CaptureContext) -> list[Finding]:
+        findings: list[Finding] = []
+        seen: set[str] = set()
+
+        def emit(key, **kw):
+            if key in seen:
+                return
+            seen.add(key)
+            kw["evidence"] = dict(kw.get("evidence") or {}, dedup_key=key)
+            kw.setdefault("category", Category.BRUTEFORCE_SCAN_FILETRANSFER)
+            findings.append(Finding(**kw))
 
-        # Response metadata can reveal risky files absent from the request URI.
         for ft in ctx.file_transfers:
             ext = (ft.extension or "").lower()
             ct = (ft.content_type or "").lower()
-            is_executable_ct = any(
-                m in ct for m in ("x-msdownload", "x-dosexec", "x-executable",
-                                  "portable-executable", "x-elf", "x-msi")
-            )
-            if ext not in _SUSPICIOUS_EXT and not is_executable_ct:
-                continue
-            key = f"ft:{ft.dst_ip}:{ft.filename or ct}"
-            if key in seen_files:
-                continue
-            seen_files.add(key)
-            label = ext or ct or "binary file"
-            findings.append(Finding(
-                title=f"Potentially harmful file download ({label})",
-                category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
-                severity=Severity.HIGH, confidence=Confidence.MEDIUM,
-                source_ip=ft.dst_ip, destination_ip=ft.src_ip, protocol=ft.protocol,
-                timestamp_first_seen=ft.timestamp, timestamp_last_seen=ft.timestamp,
-                risk_description=(
-                    "An executable or script-capable file was transferred and may represent "
-                    "a malware download."
-                ),
-                recommendation="Analyze the file in a sandbox and check its hash against threat intelligence.",
-                evidence={"filename": ft.filename, "content_type": ft.content_type,
-                          "extension": ext or None, "dedup_key": key},
-                related_packets=[p for p in [ft.packet_number] if p],
-            ))
+            name = ft.filename or ft.uri or ct or "binary"
+            executable = ext in _EXECUTABLE_EXT or any(m in ct for m in _EXECUTABLE_CT)
+            firmware = ext in _FIRMWARE_EXT or (not ext and "octet-stream" in ct and (ft.size or 0) > 64 * 1024)
+            cleartext = (ft.protocol or "") in ("http", "tftp", "ftp")
+            downloader = ft.dst_ip if ft.direction == "download" else ft.src_ip
+            server = ft.src_ip if ft.direction == "download" else ft.dst_ip
+            device_involved = ctx.is_device(downloader) or ctx.is_device(server)
+            common = dict(source_ip=downloader, destination_ip=server, protocol=ft.protocol,
+                          timestamp_first_seen=ft.timestamp, timestamp_last_seen=ft.timestamp,
+                          related_packets=[p for p in [ft.packet_number] if p],
+                          related_streams=[s for s in [ft.stream_id] if s is not None])
+            evidence = {"filename": ft.filename, "uri": ft.uri, "host": ft.host,
+                        "content_type": ft.content_type, "size": ft.size, "extension": ext or None,
+                        "direction": ft.direction}
 
-        return self._tag(findings)
+            if executable and cleartext:
+                emit(f"exec:{server}:{name}",
+                     title=f"Executable or script transferred over clear-text {ft.protocol.upper()} ({ext or ct})",
+                     severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                     risk_description=("Executable content was transferred without transport protection. It can be "
+                                       "modified on the path, and executable downloads to a device are a common "
+                                       "malware delivery vector."),
+                     recommendation="Serve executables only over TLS with signature verification; analyze the file in a sandbox and check its hash against threat intelligence.",
+                     evidence=evidence, **common)
+            elif firmware and cleartext:
+                emit(f"fw:{server}:{name}",
+                     title=f"Firmware or update image transferred over clear-text {ft.protocol.upper()}",
+                     severity=Severity.HIGH if device_involved else Severity.MEDIUM,
+                     confidence=Confidence.MEDIUM,
+                     risk_description=("An update or archive image was transferred without transport protection. "
+                                       "Unless the image is cryptographically signed and verified by the device, an "
+                                       "attacker on the path can replace it."),
+                     recommendation="Deliver updates over TLS with certificate validation and verify a vendor signature before installing.",
+                     evidence=evidence, **common)
+            elif executable:
+                emit(f"exec:{server}:{name}",
+                     title=f"Executable or script transferred ({ext or ct})",
+                     severity=Severity.MEDIUM, confidence=Confidence.LOW,
+                     risk_description="Executable content was transferred; verify that the source is trusted.",
+                     recommendation="Check the file hash against threat intelligence.",
+                     evidence=evidence, **common)
+
+        # Requests for executables that produced no dissected response.
+        for req in ctx.http_requests:
+            uri = (req.uri or "").split("?")[0]
+            ext = os.path.splitext(uri)[1].lower()
+            if ext in _EXECUTABLE_EXT:
+                emit(f"exec:{req.dst_ip}:{uri}",
+                     title=f"Executable or script requested over clear-text HTTP ({ext})",
+                     severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                     source_ip=req.src_ip, destination_ip=req.dst_ip, protocol="http", port=req.dst_port,
+                     timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
+                     risk_description=("A script or executable was requested over unencrypted HTTP and can be "
+                                       "replaced on the path."),
+                     recommendation="Serve executables only over TLS with signature verification.",
+                     evidence={"host": req.host, "uri": uri, "extension": ext},
+                     related_packets=[p for p in [req.packet_number] if p])
+        return findings

django_project/analyzer/detectors/dns_anomaly.py

@@ -1,4 +1,8 @@
-"""Detect anomalous DNS activity and potential DNS tunneling."""
+"""Detect anomalous DNS activity and potential DNS tunneling.
+
+DGA and tunneling are judged by volume and diversity of names, never by a
+single random-looking host name. Provider domains come from ``allowlist``.
+"""
 
 from __future__ import annotations
 
@@ -8,22 +12,41 @@ from typing import Iterable
 
 from ..context import CaptureContext
 from ..findings import Finding, Category, Severity, Confidence
+from .. import allowlist
 from .base import BaseDetector, register
 
 _LONG_LABEL = 40
-_ENTROPY_MIN_LEN = 12
-_HIGH_ENTROPY = 3.2
-_NXDOMAIN_RATIO = 0.3
-_MANY_SUBDOMAINS = 30
+_RANDOM_MIN_LEN = 16
+_RANDOM_ENTROPY = 3.2
+_DGA_MIN_DISTINCT = 8
+_NXDOMAIN_RATIO = 0.35
+_NXDOMAIN_MIN_RESPONSES = 20
+_MANY_SUBDOMAINS = 40
+_MANY_SUBDOMAINS_RANDOM_SHARE = 0.3
 _TUNNEL_QTYPES = {"TXT", "NULL"}
 _TXT_TUNNEL_MIN = 20
 _TXT_TUNNEL_RATIO = 0.5
 
+# Public resolvers commonly hard-coded in IoT firmware.
+_PUBLIC_RESOLVERS = {"8.8.8.8", "8.8.4.4", "1.1.1.1", "1.0.0.1", "9.9.9.9", "149.112.112.112",
+                     "208.67.222.222", "208.67.220.220", "114.114.114.114", "223.5.5.5", "223.6.6.6"}
+
+
+def _is_random_label(label: str) -> bool:
+    """Whether one DNS label looks algorithmically generated or data-encoding."""
+    if len(label) >= _LONG_LABEL:
+        return True
+    if len(label) >= _RANDOM_MIN_LEN and _entropy(label) >= _RANDOM_ENTROPY:
+        has_digit = any(c.isdigit() for c in label)
+        has_alpha = any(c.isalpha() for c in label)
+        return has_digit and has_alpha
+    return False
+
 
 @register
 class DnsAnomalyDetector(BaseDetector):
     name = "DnsAnomalyDetector"
-    description = "Detects DNS tunneling, DGA-like names, and high NXDOMAIN rates."
+    description = "Detects DNS tunneling, DGA-like names, high NXDOMAIN rates and resolver bypass."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
@@ -31,82 +54,158 @@ class DnsAnomalyDetector(BaseDetector):
             return findings
 
         nx = 0
-        total = 0
+        responses = 0
+        queries = 0
         txt_like = 0
         subdomains_per_domain: dict[str, set[str]] = defaultdict(set)
         packets_per_domain: dict[str, list] = defaultdict(list)
+        random_subs_per_domain: dict[str, set[str]] = defaultdict(set)
+        random_meta: dict[str, dict] = {}
+        long_label_hits: dict[str, dict] = {}
         nx_packets: list = []
+        nx_clients: dict[str, int] = defaultdict(int)
         txt_packets: list = []
-        flagged_random: set[str] = set()
+        resolver_use: dict[tuple, dict] = {}
 
         for q in ctx.dns_queries:
             name = (q.query_name or "").rstrip(".").lower()
-            if not name:
+            if not name or q.transport in ("mdns", "llmnr"):
                 continue
-            total += 1
-            if (q.rcode or "").upper() == "NXDOMAIN":
-                nx += 1
-                if q.packet_number:
-                    nx_packets.append(q.packet_number)
+            if q.is_response:
+                responses += 1
+                if (q.rcode or "").upper() == "NXDOMAIN":
+                    nx += 1
+                    if q.packet_number:
+                        nx_packets.append(q.packet_number)
+                    if q.dst_ip:
+                        nx_clients[q.dst_ip] += 1
+                continue
+
+            queries += 1
             if (q.qtype or "").upper() in _TUNNEL_QTYPES:
                 txt_like += 1
                 if q.packet_number:
                     txt_packets.append(q.packet_number)
+            if q.dst_ip and q.src_ip and not ctx.is_external(q.src_ip):
+                key = (q.src_ip, q.dst_ip)
+                e = resolver_use.setdefault(key, {"count": 0, "first": q.timestamp, "pkt": q.packet_number})
+                e["count"] += 1
 
             labels = name.split(".")
             reg_domain = ".".join(labels[-2:]) if len(labels) >= 2 else name
-            if len(labels) > 2:
-                subdomains_per_domain[reg_domain].add(".".join(labels[:-2]))
-                if q.packet_number:
-                    packets_per_domain[reg_domain].append(q.packet_number)
+            if allowlist.is_benign(name):
+                continue
+            sub_labels = labels[:-2]
+            if not sub_labels:
+                continue
+            subdomain = ".".join(sub_labels)
+            subdomains_per_domain[reg_domain].add(subdomain)
+            if q.packet_number:
+                packets_per_domain[reg_domain].append(q.packet_number)
+            longest = max(sub_labels, key=len)
+            if len(longest) >= _LONG_LABEL and reg_domain not in long_label_hits:
+                long_label_hits[reg_domain] = {
+                    "query_name": name, "label_len": len(longest),
+                    "packet": q.packet_number, "src": q.src_ip, "dst": q.dst_ip, "ts": q.timestamp,
+                }
+            if any(_is_random_label(lbl) for lbl in sub_labels):
+                random_subs_per_domain[reg_domain].add(subdomain)
+                random_meta.setdefault(reg_domain, {"src": q.src_ip, "dst": q.dst_ip, "ts": q.timestamp})
 
-            # Long or high-entropy labels may indicate a DGA or tunneling channel.
-            for label in labels[:-2] or labels:
-                if len(label) >= _LONG_LABEL or (
-                    len(label) >= _ENTROPY_MIN_LEN and _entropy(label) >= _HIGH_ENTROPY
-                ):
-                    if reg_domain not in flagged_random:
-                        flagged_random.add(reg_domain)
-                        findings.append(Finding(
-                            title="DNS query suspicious for DGA or tunneling activity",
-                            category=Category.DNS_ANOMALY,
-                            severity=Severity.HIGH, confidence=Confidence.MEDIUM,
-                            source_ip=q.src_ip, destination_ip=q.dst_ip, protocol="dns", port=53,
-                            timestamp_first_seen=q.timestamp, timestamp_last_seen=q.timestamp,
-                            risk_description=(
-                                "Long or random-looking subdomains are characteristic of DNS "
-                                "tunneling and malware domain-generation algorithms."
-                            ),
-                            recommendation=(
-                                "Check the domain against threat intelligence and enable DNS "
-                                "tunneling detection and egress controls."
-                            ),
-                            evidence={"query_name": name, "registered_domain": reg_domain,
-                                      "label_entropy": round(_entropy(label), 2),
-                                      "dedup_key": f"dga:{reg_domain}"},
-                            related_packets=[p for p in [q.packet_number] if p],
-                        ))
-                    break
-
-        for domain, subs in subdomains_per_domain.items():
-            if len(subs) >= _MANY_SUBDOMAINS:
+        # 1) DGA/tunneling: many distinct random subdomains under one domain.
+        for reg_domain, subs in random_subs_per_domain.items():
+            meta = random_meta.get(reg_domain, {})
+            if len(subs) < _DGA_MIN_DISTINCT:
+                if reg_domain in long_label_hits:
+                    continue
                 findings.append(Finding(
-                    title=f"Unusually many subdomains under {domain}",
+                    title="Randomly-structured DNS name observed",
                     category=Category.DNS_ANOMALY,
-                    severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                    severity=Severity.LOW, confidence=Confidence.LOW,
+                    source_ip=meta.get("src"), destination_ip=meta.get("dst"),
                     protocol="dns", port=53,
+                    timestamp_first_seen=meta.get("ts"), timestamp_last_seen=meta.get("ts"),
                     risk_description=(
-                        "A large number of unique subdomains under one domain is a strong "
-                        "indicator of DNS tunneling or exfiltration."
+                        "A DNS name with a random-looking label was queried under "
+                        f"{reg_domain}. In isolation this is a weak indicator; it becomes "
+                        "significant only alongside many such names or other anomalies."
                     ),
-                    recommendation="Review or block the domain and enable tunneling detection.",
-                    evidence={"registered_domain": domain, "unique_subdomains": len(subs),
-                              "dedup_key": f"subtunnel:{domain}"},
-                    related_packets=sorted(set(packets_per_domain.get(domain, [])))[:8],
+                    recommendation="Confirm the domain is a legitimate service; otherwise check it against threat intelligence.",
+                    evidence={"registered_domain": reg_domain,
+                              "distinct_random_subdomains": len(subs),
+                              "examples": sorted(subs)[:5],
+                              "dedup_key": f"randname:{reg_domain}"},
+                    related_packets=sorted(set(packets_per_domain.get(reg_domain, [])))[:8],
                 ))
+                continue
+            findings.append(Finding(
+                title="DNS queries consistent with DGA or tunneling activity",
+                category=Category.DNS_ANOMALY,
+                severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                source_ip=meta.get("src"), destination_ip=meta.get("dst"),
+                protocol="dns", port=53,
+                timestamp_first_seen=meta.get("ts"), timestamp_last_seen=meta.get("ts"),
+                risk_description=(
+                    f"{len(subs)} distinct random-looking subdomains were queried under "
+                    f"{reg_domain}. A high number of algorithmically generated names is a "
+                    "strong indicator of a domain-generation algorithm or DNS tunneling."
+                ),
+                recommendation="Check the domain against threat intelligence and enable DNS tunneling detection and egress controls.",
+                evidence={"registered_domain": reg_domain,
+                          "distinct_random_subdomains": len(subs),
+                          "examples": sorted(subs)[:5],
+                          "dedup_key": f"dga:{reg_domain}"},
+                related_packets=sorted(set(packets_per_domain.get(reg_domain, [])))[:8],
+            ))
 
-        # TXT and NULL records can carry the large payloads used by DNS tunnels.
-        if txt_like >= _TXT_TUNNEL_MIN and total and (txt_like / total) >= _TXT_TUNNEL_RATIO:
+        # 2) Long-label tunneling.
+        for reg_domain, hit in long_label_hits.items():
+            if len(random_subs_per_domain.get(reg_domain, ())) >= _DGA_MIN_DISTINCT:
+                continue
+            findings.append(Finding(
+                title="Overlong DNS label suggests tunneling",
+                category=Category.DNS_ANOMALY,
+                severity=Severity.MEDIUM, confidence=Confidence.LOW,
+                source_ip=hit.get("src"), destination_ip=hit.get("dst"),
+                protocol="dns", port=53,
+                timestamp_first_seen=hit.get("ts"), timestamp_last_seen=hit.get("ts"),
+                risk_description=(
+                    f"A DNS label of {hit['label_len']} characters was observed under "
+                    f"{reg_domain}. Overlong labels are used to encode data in DNS tunnels."
+                ),
+                recommendation="Review the destination domain and enable tunneling detection.",
+                evidence={"registered_domain": reg_domain,
+                          "query_name": hit["query_name"], "label_length": hit["label_len"],
+                          "dedup_key": f"longlabel:{reg_domain}"},
+                related_packets=[p for p in [hit.get("packet")] if p],
+            ))
+
+        # 3) Many unique subdomains; HIGH only when many of them look generated.
+        for domain, subs in subdomains_per_domain.items():
+            if len(subs) < _MANY_SUBDOMAINS:
+                continue
+            random_share = len(random_subs_per_domain.get(domain, ())) / len(subs)
+            strong = random_share >= _MANY_SUBDOMAINS_RANDOM_SHARE
+            findings.append(Finding(
+                title=f"Unusually many subdomains under {domain}",
+                category=Category.DNS_ANOMALY,
+                severity=Severity.HIGH if strong else Severity.LOW,
+                confidence=Confidence.MEDIUM if strong else Confidence.LOW,
+                protocol="dns", port=53,
+                risk_description=(
+                    f"{len(subs)} unique subdomains were queried under {domain}"
+                    + (f", {round(random_share * 100)} % of them random-looking, a strong indicator of DNS tunneling or exfiltration."
+                       if strong else ". Large services use many host names; this is only suspicious if the domain is unexpected for the device.")
+                ),
+                recommendation="Review the domain; block it and enable tunneling detection if it is not an expected service.",
+                evidence={"registered_domain": domain, "unique_subdomains": len(subs),
+                          "random_share": round(random_share, 2), "examples": sorted(subs)[:5],
+                          "dedup_key": f"subtunnel:{domain}"},
+                related_packets=sorted(set(packets_per_domain.get(domain, [])))[:8],
+            ))
+
+        # 4) TXT/NULL abuse.
+        if txt_like >= _TXT_TUNNEL_MIN and queries and (txt_like / queries) >= _TXT_TUNNEL_RATIO:
             findings.append(Finding(
                 title="Unusually many TXT or NULL DNS queries",
                 category=Category.DNS_ANOMALY,
@@ -116,31 +215,50 @@ class DnsAnomalyDetector(BaseDetector):
                     "TXT and NULL records can carry large payloads and are commonly abused "
                     "for DNS tunneling or data exfiltration."
                 ),
-                recommendation=(
-                    "Review the requesting source and destination domain, then enforce DNS "
-                    "tunneling detection and egress controls."
-                ),
-                evidence={"txt_null_queries": txt_like, "total_queries": total,
-                          "ratio": round(txt_like / total, 2), "dedup_key": "txt_tunnel"},
+                recommendation="Review the requesting source and destination domain, then enforce DNS tunneling detection and egress controls.",
+                evidence={"txt_null_queries": txt_like, "total_queries": queries,
+                          "ratio": round(txt_like / queries, 2), "dedup_key": "txt_tunnel"},
                 related_packets=sorted(set(txt_packets))[:8],
             ))
 
-        if total >= 20 and (nx / total) >= _NXDOMAIN_RATIO:
+        # 5) High NXDOMAIN rate, measured over responses only.
+        if responses >= _NXDOMAIN_MIN_RESPONSES and (nx / responses) >= _NXDOMAIN_RATIO:
+            top_client = max(nx_clients.items(), key=lambda kv: kv[1])[0] if nx_clients else None
             findings.append(Finding(
                 title="High NXDOMAIN rate",
                 category=Category.DNS_ANOMALY,
                 severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
+                source_ip=top_client,
                 protocol="dns", port=53,
                 risk_description=(
-                    "Repeated NXDOMAIN responses may indicate malware testing generated "
-                    "command-and-control domains."
+                    "A large share of DNS responses were NXDOMAIN. Malware probing generated "
+                    "command-and-control domains produces this pattern; so do stale configurations."
                 ),
-                recommendation="Review the requesting source and investigate it for malware.",
-                evidence={"nxdomain": nx, "total_queries": total,
-                          "ratio": round(nx / total, 2), "dedup_key": "nxdomain_rate"},
+                recommendation="Review the requesting source and the failed names.",
+                evidence={"nxdomain": nx, "responses": responses,
+                          "ratio": round(nx / responses, 2), "dedup_key": "nxdomain_rate"},
                 related_packets=sorted(set(nx_packets))[:8],
             ))
 
+        # 6) Direct use of public resolvers (bypasses the local resolver and its policies).
+        for (client, resolver), e in resolver_use.items():
+            if resolver in _PUBLIC_RESOLVERS or (ctx.is_external(resolver) and ctx.is_device(client)):
+                findings.append(Finding(
+                    title=f"DNS sent directly to an external resolver ({resolver})",
+                    category=Category.DNS_ANOMALY,
+                    severity=Severity.LOW, confidence=Confidence.HIGH,
+                    source_ip=client, destination_ip=resolver, protocol="dns", port=53,
+                    timestamp_first_seen=e["first"], timestamp_last_seen=e["first"],
+                    risk_description=(
+                        "The host resolves names through a hard-coded external resolver instead of the "
+                        "one assigned by the network. Local DNS filtering and monitoring are bypassed and "
+                        "the queries are visible to a third party."
+                    ),
+                    recommendation="Use the DHCP-assigned resolver, or DNS over TLS/HTTPS if privacy is required.",
+                    evidence={"queries": e["count"], "dedup_key": f"extresolver:{client}:{resolver}"},
+                    related_packets=[p for p in [e["pkt"]] if p],
+                ))
+
         return self._tag(findings)
 
 

django_project/analyzer/detectors/exfiltration.py

@@ -1,4 +1,4 @@
-"""Detect potential data exfiltration."""
+"""Detect unusual outbound data volumes."""
 
 from __future__ import annotations
 
@@ -7,13 +7,15 @@ from typing import Iterable
 
 from ..context import CaptureContext
 from ..findings import Finding, Category, Severity, Confidence
+from .. import allowlist
 from .base import BaseDetector, register
 
 # Conservative heuristic thresholds.
 _LARGE_UPLOAD_BYTES = 5 * 1024 * 1024
 _AGG_UPLOAD_BYTES = 20 * 1024 * 1024
 _MANY_SMALL_TRANSFERS = 50
-_DNS_TXT_VOLUME = 100
+# Uploads must clearly dominate the flow; browsing and streaming are download-heavy.
+_UPLOAD_RATIO = 3.0
 
 
 @register
@@ -37,7 +39,8 @@ class ExfiltrationDetector(BaseDetector):
             if not (conv.src_ip and ctx.is_external(conv.dst_ip)
                     and not ctx.is_external(conv.src_ip)):
                 continue
-            up = conv.bytes_src_to_dst or conv.bytes
+            up = conv.bytes_src_to_dst
+            down = conv.bytes_dst_to_src
             per_dest_bytes[conv.dst_ip] += up
             per_dest_count[conv.dst_ip] += 1
             src_for_dest[conv.dst_ip] = conv.src_ip
@@ -50,76 +53,69 @@ class ExfiltrationDetector(BaseDetector):
             if conv.last_seen is not None:
                 last_seen[conv.dst_ip] = max(last_seen.get(conv.dst_ip, conv.last_seen), conv.last_seen)
 
-            if up >= _LARGE_UPLOAD_BYTES:
+            if up >= _LARGE_UPLOAD_BYTES and up >= _UPLOAD_RATIO * max(down, 1):
+                benign = allowlist.benign_name_for_ip(ctx, conv.dst_ip)
+                names = sorted(ctx.names_for_ip(conv.dst_ip))[:5]
                 findings.append(Finding(
                     title="Large outbound transfer to an external target",
                     category=Category.EXFILTRATION,
-                    severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                    severity=Severity.MEDIUM if benign else Severity.HIGH,
+                    confidence=Confidence.LOW if benign else Confidence.MEDIUM,
                     source_ip=conv.src_ip, destination_ip=conv.dst_ip,
                     protocol=conv.protocol, port=conv.dst_port,
                     timestamp_first_seen=conv.first_seen, timestamp_last_seen=conv.last_seen,
                     risk_description=(
-                        "A large amount of data left the device for an external target, "
-                        "which may indicate data loss."
+                        "A large amount of data left the network for an external target in one "
+                        "upload-dominated flow. "
+                        + (f"The target belongs to an expected provider ({benign}), e.g. a backup or sync service."
+                           if benign else "Verify what was sent and whether the destination is expected.")
                     ),
                     recommendation="Validate the destination, content, and authorization.",
-                    evidence={"uploaded_bytes": up,
+                    evidence={"uploaded_bytes": up, "downloaded_bytes": down,
+                              "encrypted": conv.is_encrypted, "target_names": names,
                               "dedup_key": f"bigup:{conv.src_ip}:{conv.dst_ip}:{conv.dst_port}"},
                     related_packets=list(conv.packet_numbers),
                     related_streams=[s for s in [conv.stream_id] if s is not None],
                 ))
 
         for dest, total in per_dest_bytes.items():
+            benign = allowlist.benign_name_for_ip(ctx, dest)
+            names = sorted(ctx.names_for_ip(dest))[:5]
             if total >= _AGG_UPLOAD_BYTES:
                 findings.append(Finding(
                     title="High aggregate upload volume to an external target",
                     category=Category.EXFILTRATION,
-                    severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                    severity=Severity.MEDIUM if benign else Severity.HIGH,
+                    confidence=Confidence.LOW if benign else Confidence.MEDIUM,
                     source_ip=src_for_dest.get(dest), destination_ip=dest,
                     timestamp_first_seen=first_seen.get(dest),
                     timestamp_last_seen=last_seen.get(dest),
                     risk_description=(
                         "A high total volume was sent to one external destination across "
-                        "multiple connections, which may indicate gradual exfiltration."
+                        "multiple connections, which may indicate gradual data loss."
+                        + (f" The destination resolves to {benign}." if benign else "")
                     ),
-                    recommendation="Review the destination and enforce egress filtering or DLP.",
-                    evidence={"uploaded_bytes_total": total,
+                    recommendation="Review the destination and enforce egress filtering.",
+                    evidence={"uploaded_bytes_total": total, "target_names": names,
                               "dedup_key": f"aggup:{dest}"},
                     related_packets=sorted(set(packets_for_dest.get(dest, []))),
                     related_streams=sorted(set(streams_for_dest.get(dest, []))),
                 ))
-            elif per_dest_count[dest] >= _MANY_SMALL_TRANSFERS:
+            elif per_dest_count[dest] >= _MANY_SMALL_TRANSFERS and not benign:
                 findings.append(Finding(
-                    title="Many small outbound transfers to one target",
+                    title="Many small outbound connections to one target",
                     category=Category.EXFILTRATION,
-                    severity=Severity.MEDIUM, confidence=Confidence.LOW,
+                    severity=Severity.LOW, confidence=Confidence.LOW,
                     source_ip=src_for_dest.get(dest), destination_ip=dest,
                     risk_description=(
-                        "Many small outbound connections to one target may indicate "
-                        "segmented exfiltration or beaconing."
+                        "Many short outbound connections to one target may indicate "
+                        "segmented uploads or polling."
                     ),
                     recommendation="Investigate the connection pattern and destination.",
-                    evidence={"transfer_count": per_dest_count[dest],
+                    evidence={"transfer_count": per_dest_count[dest], "target_names": names,
                               "dedup_key": f"manyup:{dest}"},
                     related_packets=sorted(set(packets_for_dest.get(dest, []))),
                     related_streams=sorted(set(streams_for_dest.get(dest, []))),
                 ))
 
-        # DNS volume is a weak signal; the DNS detector provides deeper evidence.
-        if len(ctx.dns_queries) >= _DNS_TXT_VOLUME:
-            findings.append(Finding(
-                title="Very high DNS query volume",
-                category=Category.EXFILTRATION,
-                severity=Severity.MEDIUM, confidence=Confidence.LOW,
-                protocol="dns",
-                risk_description=(
-                    "Unusually high DNS volume may indicate DNS tunneling or exfiltration."
-                ),
-                recommendation="Review DNS logs and correlated DNS anomaly findings.",
-                evidence={"dns_query_count": len(ctx.dns_queries), "dedup_key": "dns_volume"},
-                related_packets=sorted(
-                    {q.packet_number for q in ctx.dns_queries if q.packet_number}
-                )[:8],
-            ))
-
         return self._tag(findings)

django_project/analyzer/detectors/insecure_protocols.py

@@ -1,91 +1,149 @@
-"""Detect insecure or unexpected legacy protocols."""
+"""Detect clear-text, legacy or unexpectedly exposed protocols.
+
+Based on what tshark dissected on a flow; port rules are only a fallback.
+"""
 
 from __future__ import annotations
 
 from collections import defaultdict
 from typing import Iterable
 
-from ..context import CaptureContext
+from ..context import CaptureContext, Conversation
 from ..findings import Finding, Category, Severity, Confidence
 from .base import BaseDetector, register
 
-# Port to protocol name, severity, risk, and recommendation.
-_RISKY_PORTS = {
-    21:    ("FTP", Severity.HIGH, "FTP transmits data and credentials in clear text.",
-            "Replace FTP with FTPS or SFTP."),
-    23:    ("Telnet", Severity.CRITICAL, "Telnet transmits all content in clear text.",
-            "Disable Telnet and use SSH."),
-    69:    ("TFTP", Severity.HIGH, "TFTP has no authentication or encryption.",
-            "Avoid TFTP or restrict it to an isolated network."),
-    80:    ("HTTP", Severity.MEDIUM, "HTTP content is unencrypted and observable.",
-            "Require HTTPS."),
-    110:   ("POP3", Severity.HIGH, "POP3 without TLS exposes messages and passwords.",
-            "Use POP3S or STARTTLS."),
-    143:   ("IMAP", Severity.HIGH, "IMAP without TLS exposes messages and passwords.",
-            "Use IMAPS or STARTTLS."),
-    161:   ("SNMP", Severity.MEDIUM, "SNMP v1/v2c uses clear-text community strings.",
-            "Use SNMPv3 with authentication and privacy."),
-    389:   ("LDAP", Severity.HIGH, "LDAP without TLS exposes directory credentials.",
-            "Use LDAPS or StartTLS."),
-    512:   ("rexec", Severity.HIGH, "Remote shell services are insecure and unencrypted.",
-            "Disable the service and use SSH."),
-    513:   ("rlogin", Severity.HIGH, "Remote login services are insecure and unencrypted.",
-            "Disable the service and use SSH."),
-    1433:  ("MSSQL", Severity.MEDIUM, "The database service may be exposed without TLS.",
-            "Require TLS and restrict access."),
-    3306:  ("MySQL", Severity.MEDIUM, "The database service may be exposed without TLS.",
-            "Require TLS and restrict access."),
-    6379:  ("Redis", Severity.HIGH, "Redis may be exposed without authentication or encryption.",
-            "Enable authentication and restrict network exposure."),
-    11211: ("Memcached", Severity.HIGH, "Memcached can be abused for amplification.",
-            "Bind locally and restrict access."),
-    27017: ("MongoDB", Severity.HIGH, "MongoDB without authentication or TLS exposes data.",
-            "Enable authentication and TLS, then restrict access."),
+# Dissected protocol -> display name, severity, risk, recommendation.
+_CLEARTEXT_PROTOCOLS = {
+    "telnet": ("Telnet", Severity.CRITICAL, "Telnet transmits commands and credentials in clear text.",
+               "Disable Telnet and use SSH."),
+    "ftp":    ("FTP", Severity.HIGH, "FTP transmits data and credentials in clear text.",
+               "Replace FTP with FTPS or SFTP."),
+    "tftp":   ("TFTP", Severity.HIGH, "TFTP has no authentication or encryption; it is often used for firmware or configuration transfer.",
+               "Avoid TFTP or restrict it to an isolated provisioning network."),
+    "http":   ("HTTP", Severity.MEDIUM, "HTTP content is unencrypted and can be read or modified on the path.",
+               "Require HTTPS for all device and cloud communication."),
+    "pop":    ("POP3", Severity.HIGH, "POP3 without TLS exposes messages and passwords.",
+               "Use POP3S or STARTTLS."),
+    "imap":   ("IMAP", Severity.HIGH, "IMAP without TLS exposes messages and passwords.",
+               "Use IMAPS or STARTTLS."),
+    "smtp":   ("SMTP", Severity.MEDIUM, "SMTP without TLS exposes message content and possibly credentials.",
+               "Use STARTTLS or submission over TLS."),
+    "snmp":   ("SNMP", Severity.MEDIUM, "SNMP v1/v2c uses clear-text community strings.",
+               "Use SNMPv3 with authentication and privacy."),
+    "ldap":   ("LDAP", Severity.HIGH, "LDAP without TLS exposes directory credentials.",
+               "Use LDAPS or StartTLS."),
+    "mqtt":   ("MQTT", Severity.HIGH, "MQTT without TLS exposes topics, payloads and any username/password to observers and allows message injection.",
+               "Use MQTT over TLS (8883) with server verification and per-device credentials or certificates."),
+    "coap":   ("CoAP", Severity.HIGH, "CoAP without DTLS has no confidentiality or integrity protection.",
+               "Use CoAP over DTLS (coaps) or OSCORE."),
+    "rtsp":   ("RTSP", Severity.MEDIUM, "RTSP control and the media stream are unencrypted; video and credentials can be intercepted.",
+               "Tunnel RTSP through TLS or a VPN and require authentication."),
+    "mbtcp":  ("Modbus/TCP", Severity.HIGH, "Modbus/TCP has no authentication; any host on the network can read or write registers.",
+               "Isolate the control network and use Modbus/TCP Security or a gateway with authentication."),
+    "sip":    ("SIP", Severity.MEDIUM, "SIP signalling without TLS exposes call metadata and digest challenges.",
+               "Use SIP over TLS and SRTP."),
+    "vnc":    ("VNC", Severity.HIGH, "VNC sessions are unencrypted and often weakly authenticated.",
+               "Tunnel VNC through SSH or a VPN."),
+    "xmpp":   ("XMPP", Severity.MEDIUM, "XMPP without TLS exposes messages.",
+               "Require STARTTLS."),
+    "irc":    ("IRC", Severity.MEDIUM, "IRC without TLS exposes messages; IRC is also a classic botnet control channel.",
+               "Verify the purpose of the connection and require TLS."),
+    "nbns":   ("NetBIOS name service", Severity.LOW, "NetBIOS name resolution can be answered by any host and is used for credential relay attacks.",
+               "Disable NetBIOS over TCP/IP where it is not required."),
+    "llmnr":  ("LLMNR", Severity.LOW, "LLMNR queries can be answered by any host and are used for credential relay attacks.",
+               "Disable LLMNR."),
+    "syslog": ("Syslog", Severity.LOW, "Syslog over UDP is unencrypted and unauthenticated.",
+               "Use syslog over TLS."),
 }
 
+# Port fallback for flows carrying data that no dissector recognised.
+_RISKY_PORTS = {
+    23:    ("Telnet (port 23)", Severity.CRITICAL, "Traffic on the Telnet port was not dissected but carries application data.",
+            "Disable Telnet and use SSH."),
+    512:   ("rexec", Severity.HIGH, "Remote shell services are insecure and unencrypted.", "Disable the service and use SSH."),
+    513:   ("rlogin", Severity.HIGH, "Remote login services are insecure and unencrypted.", "Disable the service and use SSH."),
+    514:   ("rsh", Severity.HIGH, "Remote shell services are insecure and unencrypted.", "Disable the service and use SSH."),
+    1433:  ("MSSQL", Severity.MEDIUM, "The database service may be exposed without TLS.", "Require TLS and restrict access."),
+    3306:  ("MySQL", Severity.MEDIUM, "The database service may be exposed without TLS.", "Require TLS and restrict access."),
+    5432:  ("PostgreSQL", Severity.MEDIUM, "The database service may be exposed without TLS.", "Require TLS and restrict access."),
+    6379:  ("Redis", Severity.HIGH, "Redis may be exposed without authentication or encryption.", "Enable authentication and restrict network exposure."),
+    11211: ("Memcached", Severity.HIGH, "Memcached can be abused for amplification and exposes cached data.", "Bind locally and restrict access."),
+    27017: ("MongoDB", Severity.HIGH, "MongoDB without authentication or TLS exposes data.", "Enable authentication and TLS, then restrict access."),
+    5900:  ("VNC (port 5900)", Severity.HIGH, "VNC sessions are unencrypted and often weakly authenticated.", "Tunnel VNC through SSH or a VPN."),
+    1883:  ("MQTT (port 1883)", Severity.HIGH, "Traffic on the MQTT port was not dissected; unencrypted MQTT is likely.", "Use MQTT over TLS (8883)."),
+    502:   ("Modbus/TCP (port 502)", Severity.HIGH, "Traffic on the Modbus port was not dissected; unauthenticated control traffic is likely.", "Isolate the control network."),
+}
+
+_ENCRYPTED_MARKERS = {"tls", "ssl", "dtls", "quic", "ssh"}
+
 
 @register
 class InsecureProtocolDetector(BaseDetector):
     name = "InsecureProtocolDetector"
-    description = "Detects clear-text and legacy protocols on known service ports."
+    description = "Detects clear-text and legacy protocols based on dissected traffic, plus exposed device services."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
 
-        # Aggregate observed use of known risky service ports.
-        used: dict[int, dict] = defaultdict(lambda: {"src": None, "dst": None, "proto": None,
-                                                     "first": None, "last": None, "packets": 0,
-                                                     "pkts": [], "streams": set()})
-        for conv in ctx.conversations:
-            for port in (conv.dst_port, conv.src_port):
-                if port in _RISKY_PORTS:
-                    e = used[port]
-                    e["src"] = conv.src_ip
-                    e["dst"] = conv.dst_ip
-                    e["proto"] = conv.protocol
-                    e["packets"] += conv.packets
-                    if conv.packet_numbers:
-                        e["pkts"].extend(conv.packet_numbers[:4])
-                    if conv.stream_id is not None:
-                        e["streams"].add(conv.stream_id)
-                    if conv.first_seen is not None:
-                        e["first"] = conv.first_seen if e["first"] is None else min(e["first"], conv.first_seen)
-                    if conv.last_seen is not None:
-                        e["last"] = conv.last_seen if e["last"] is None else max(e["last"], conv.last_seen)
+        # Aggregate per (protocol, server, port) so every exposed service is listed once.
+        used: dict[tuple, dict] = {}
 
-        for port, e in used.items():
-            name, sev, risk, rec = _RISKY_PORTS[port]
+        def note(key, conv: Conversation, client, server):
+            e = used.setdefault(key, {"clients": set(), "packets": 0, "bytes": 0, "first": None,
+                                      "last": None, "pkts": [], "streams": set(), "server": server})
+            e["clients"].add(client)
+            e["packets"] += conv.packets
+            e["bytes"] += conv.payload_bytes
+            e["pkts"].extend(conv.packet_numbers[:4])
+            if conv.stream_id is not None:
+                e["streams"].add(conv.stream_id)
+            if conv.first_seen is not None:
+                e["first"] = conv.first_seen if e["first"] is None else min(e["first"], conv.first_seen)
+            if conv.last_seen is not None:
+                e["last"] = conv.last_seen if e["last"] is None else max(e["last"], conv.last_seen)
+
+        for conv in ctx.conversations:
+            if conv.payload_bytes <= 0:
+                continue
+            if conv.app_protocols & _ENCRYPTED_MARKERS:
+                continue  # STARTTLS upgraded flows or TLS on a non-standard port.
+            matched = False
+            for proto in conv.app_protocols:
+                if proto in _CLEARTEXT_PROTOCOLS:
+                    if proto in ("nbns", "llmnr") and not conv.answered:
+                        continue
+                    note((proto, conv.dst_ip, conv.dst_port), conv, conv.src_ip, conv.dst_ip)
+                    matched = True
+            if not matched and not conv.app_protocols and conv.dst_port in _RISKY_PORTS and conv.answered:
+                note((f"port:{conv.dst_port}", conv.dst_ip, conv.dst_port), conv, conv.src_ip, conv.dst_ip)
+
+        for (proto, server, port), e in used.items():
+            if proto.startswith("port:"):
+                name, sev, risk, rec = _RISKY_PORTS[int(proto.split(":")[1])]
+                proto_label = name.split(" ")[0].lower()
+            else:
+                name, sev, risk, rec = _CLEARTEXT_PROTOCOLS[proto]
+                proto_label = proto
+            clients = sorted(c for c in e["clients"] if c)
+            device_is_server = ctx.is_device(server)
+            device_is_client = any(ctx.is_device(c) for c in clients)
+            role = "offered by the device" if device_is_server else "used by the device" if device_is_client else None
+            title = f"Clear-text protocol: {name} on {server}:{port}"
+            if role:
+                title += f" ({role})"
             findings.append(Finding(
-                title=f"Insecure protocol: {name} (port {port})",
+                title=title,
                 category=Category.INSECURE_PROTOCOL,
                 severity=sev, confidence=Confidence.HIGH,
-                source_ip=e["src"], destination_ip=e["dst"],
-                protocol=name.lower(), port=port,
+                source_ip=clients[0] if clients else None, destination_ip=server,
+                protocol=proto_label, port=port,
                 timestamp_first_seen=e["first"], timestamp_last_seen=e["last"],
                 risk_description=risk,
                 recommendation=rec,
-                evidence={"port": port, "packets": e["packets"],
-                          "dedup_key": f"insecure:{port}:{e['dst']}"},
+                evidence={"port": port, "packets": e["packets"], "payload_bytes": e["bytes"],
+                          "clients": clients[:10], "client_count": len(clients),
+                          "external_server": ctx.is_external(server),
+                          "dedup_key": f"insecure:{proto}:{server}:{port}"},
                 related_packets=sorted(set(e["pkts"]))[:8],
                 related_streams=sorted(e["streams"])[:8],
             ))
@@ -104,4 +162,61 @@ class InsecureProtocolDetector(BaseDetector):
                 evidence={"smb1_packets": smb1, "dedup_key": "smbv1"},
             ))
 
+        # SSH protocol version 1 banners.
+        for s in ctx.service_events:
+            if s.kind == "ssh_banner" and str(s.value or "").upper().startswith("SSH-1."):
+                findings.append(Finding(
+                    title="SSH protocol version 1 offered",
+                    category=Category.INSECURE_PROTOCOL,
+                    severity=Severity.HIGH, confidence=Confidence.HIGH,
+                    source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="ssh", port=s.dst_port,
+                    timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
+                    risk_description="SSH-1 has known cryptographic weaknesses and must not be used.",
+                    recommendation="Disable SSH-1 and allow only SSH-2 with modern algorithms.",
+                    evidence={"banner": s.value, "dedup_key": f"ssh1:{s.src_ip}"},
+                    related_packets=[p for p in [s.packet_number] if p],
+                ))
+
+        # Services the device under test exposes on the network (completeness, INFO).
+        if ctx.device_ip:
+            offered: dict[int, set[str]] = defaultdict(set)
+            for conv in ctx.conversations:
+                if conv.dst_ip == ctx.device_ip and conv.answered and conv.dst_port:
+                    offered[conv.dst_port] |= conv.app_protocols
+            for port in sorted(offered):
+                if port in (67, 68, 5353, 1900, 5355, 137, 138):
+                    continue  # Broadcast/multicast infrastructure, not a listening service.
+                protos = sorted(p for p in offered[port] if p not in ("x509ce", "x509sat", "pkix1implicit", "tcp.segments"))
+                findings.append(Finding(
+                    title=f"Device offers a network service on port {port}",
+                    category=Category.INSECURE_PROTOCOL,
+                    severity=Severity.INFO, confidence=Confidence.HIGH,
+                    destination_ip=ctx.device_ip, port=port,
+                    protocol=(protos[0] if protos else None),
+                    risk_description=(
+                        "The device accepted connections on this port. Every listening service "
+                        "is attack surface and should be known and justified."
+                    ),
+                    recommendation="Confirm the service is required, authenticated and encrypted; disable it otherwise.",
+                    evidence={"port": port, "protocols": protos, "dedup_key": f"devsvc:{port}"},
+                ))
+            # UPnP/SSDP announcements from the device.
+            ssdp = [s for s in ctx.service_events if s.protocol == "ssdp" and s.src_ip == ctx.device_ip]
+            if ssdp:
+                findings.append(Finding(
+                    title="Device announces itself via UPnP/SSDP",
+                    category=Category.INSECURE_PROTOCOL,
+                    severity=Severity.LOW, confidence=Confidence.HIGH,
+                    source_ip=ctx.device_ip, protocol="ssdp", port=1900,
+                    timestamp_first_seen=ssdp[0].timestamp, timestamp_last_seen=ssdp[-1].timestamp,
+                    risk_description=(
+                        "UPnP exposes a device description and control endpoints to every host on "
+                        "the network; UPnP control interfaces are frequently unauthenticated."
+                    ),
+                    recommendation="Disable UPnP if not required, or restrict it to trusted hosts.",
+                    evidence={"announcements": len(ssdp), "location": ssdp[0].value, "server": ssdp[0].detail,
+                              "dedup_key": "device_ssdp"},
+                    related_packets=sorted({s.packet_number for s in ssdp if s.packet_number})[:8],
+                ))
+
         return self._tag(findings)

django_project/analyzer/detectors/malware_beaconing.py

@@ -1,4 +1,4 @@
-"""Detect malware, command-and-control, and beaconing behavior."""
+"""Detect command-and-control-like beaconing and suspicious user agents."""
 
 from __future__ import annotations
 
@@ -8,6 +8,7 @@ from typing import Iterable, Optional
 
 from ..context import CaptureContext
 from ..findings import Finding, Category, Severity, Confidence
+from .. import allowlist
 from .base import BaseDetector, register
 
 # Scanner and attack-tool user agents provide a strong signal.
@@ -19,10 +20,16 @@ _SCANNER_UA = (
 _SCRIPTING_UA = (
     "curl", "wget", "python-requests", "powershell", "libwww",
 )
-# Common mobile and embedded clients are intentionally excluded to limit false positives.
 
 _MIN_BEACONS = 5
 _MAX_CV = 0.25
+# Beacons repeat on a human time scale; sub-second pacing is just bulk traffic.
+_MIN_INTERVAL_S = 5.0
+_MAX_INTERVAL_S = 6 * 3600
+# Infrastructure protocols poll periodically by design.
+_PERIODIC_PORTS = {123, 53, 67, 68, 1900, 5353, 5355}
+# Bulk flows are downloads or streams, not beacons.
+_BULK_BYTES = 2 * 1024 * 1024
 
 
 @register
@@ -32,10 +39,8 @@ class MalwareBeaconingDetector(BaseDetector):
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
-
         findings.extend(self._detect_beaconing(ctx))
         findings.extend(self._detect_suspicious_ua(ctx))
-
         return self._tag(findings)
 
     def _detect_beaconing(self, ctx: CaptureContext) -> list[Finding]:
@@ -43,18 +48,20 @@ class MalwareBeaconingDetector(BaseDetector):
         findings: list[Finding] = []
         reported: set[tuple] = set()
 
-        # First evaluate multiple timestamps captured within a single flow.
+        # First evaluate client-side activity inside a single long-lived flow.
         for conv in ctx.conversations:
             ext_ip = _external_of(ctx, conv.src_ip, conv.dst_ip)
-            if not ext_ip:
+            if not ext_ip or conv.bytes >= _BULK_BYTES:
+                continue
+            port = conv.dst_port if ext_ip == conv.dst_ip else conv.src_port
+            if port in _PERIODIC_PORTS:
                 continue
             ts = sorted(t for t in conv.timestamps if t is not None)
             if len(ts) < _MIN_BEACONS:
                 continue
-            internal = conv.dst_ip if ext_ip == conv.src_ip else conv.src_ip
-            port = conv.src_port if ext_ip == conv.src_ip else conv.dst_port
+            internal = conv.src_ip if ext_ip == conv.dst_ip else conv.dst_ip
             f = self._beacon_finding(
-                ts, src=internal, dst=ext_ip, protocol=conv.protocol, port=port,
+                ctx, _cluster_bursts(ts), src=internal, dst=ext_ip, protocol=conv.protocol, port=port,
                 streams=[conv.stream_id] if conv.stream_id is not None else [],
                 packets=list(conv.packet_numbers),
             )
@@ -68,18 +75,15 @@ class MalwareBeaconingDetector(BaseDetector):
         for conv in ctx.conversations:
             if ctx.is_external(conv.dst_ip) and not ctx.is_external(conv.src_ip):
                 internal, external, port = conv.src_ip, conv.dst_ip, conv.dst_port
-            elif ctx.is_external(conv.src_ip) and not ctx.is_external(conv.dst_ip):
-                internal, external, port = conv.dst_ip, conv.src_ip, conv.src_port
             else:
                 continue
-            if not (internal and external) or conv.first_seen is None:
+            if not (internal and external) or conv.first_seen is None or port in _PERIODIC_PORTS:
                 continue
             key = (internal, external, port)
             events[key].append(conv.first_seen)
             m = meta.setdefault(key, {"protocol": conv.protocol, "streams": set(), "packets": []})
             if conv.stream_id is not None:
                 m["streams"].add(conv.stream_id)
-            # One representative packet per connection keeps evidence compact.
             if conv.packet_numbers:
                 m["packets"].append(conv.packet_numbers[0])
 
@@ -88,7 +92,7 @@ class MalwareBeaconingDetector(BaseDetector):
                 continue
             internal, external, port = key
             f = self._beacon_finding(
-                sorted(first_seens), src=internal, dst=external,
+                ctx, sorted(first_seens), src=internal, dst=external,
                 protocol=meta[key]["protocol"], port=port,
                 streams=sorted(meta[key]["streams"]),
                 packets=meta[key]["packets"],
@@ -99,39 +103,55 @@ class MalwareBeaconingDetector(BaseDetector):
 
         return findings
 
-    def _beacon_finding(self, ts: list[float], *, src, dst, protocol, port,
+    def _beacon_finding(self, ctx: CaptureContext, ts: list[float], *, src, dst, protocol, port,
                         streams, packets=None, connection_based: bool = False) -> Optional[Finding]:
         intervals = [b - a for a, b in zip(ts, ts[1:]) if (b - a) > 0]
         if len(intervals) < _MIN_BEACONS - 1:
             return None
         mean = statistics.mean(intervals)
-        if mean <= 0:
+        if mean < _MIN_INTERVAL_S or mean > _MAX_INTERVAL_S:
             return None
         cv = statistics.pstdev(intervals) / mean
         if cv > _MAX_CV:
             return None
         conf = Confidence.MEDIUM if cv > 0.1 else Confidence.HIGH
+
+        names = sorted(ctx.names_for_ip(dst))
+        benign = allowlist.benign_name_for_ip(ctx, dst)
+        # Known provider: expected keep-alive. Unnamed address: worth a look.
+        if benign:
+            sev, conf = Severity.LOW, Confidence.LOW
+            note = f"The target resolves to an expected provider ({benign}); this is most likely keep-alive or telemetry traffic."
+        elif names:
+            sev = Severity.MEDIUM
+            note = "The target has a DNS name; verify that the service is expected for this device."
+        else:
+            sev = Severity.HIGH
+            note = "The target was contacted by IP address without a DNS lookup, which is unusual for legitimate cloud services."
+
         evidence = {
             "beacon_count": len(ts),
             "interval_seconds_mean": round(mean, 2),
             "interval_cv": round(cv, 3),
             "signal": "distinct_connections" if connection_based else "single_flow",
+            "target_names": names[:5],
             "dedup_key": f"beacon:{src}:{dst}:{port}",
         }
         return Finding(
-            title="Periodic beaconing to an external target",
+            title="Periodic connections to an external target",
             category=Category.MALWARE_BEACONING,
-            severity=Severity.HIGH, confidence=conf,
+            severity=sev, confidence=conf,
             source_ip=src, destination_ip=dst,
             protocol=protocol, port=port,
             timestamp_first_seen=ts[0], timestamp_last_seen=ts[-1],
             risk_description=(
-                "The device contacts an external target at highly regular intervals, "
-                "which is characteristic of command-and-control beaconing."
+                "The host contacts an external target at highly regular intervals "
+                f"(about every {round(mean)} s). Regular check-ins are characteristic of "
+                "command-and-control beaconing, but also of legitimate telemetry. " + note
             ),
             recommendation=(
-                "Check the target against threat intelligence, isolate the device, and "
-                "investigate it for compromise."
+                "Confirm the target belongs to the device vendor or an expected service. "
+                "If unknown, check it against threat intelligence and isolate the device."
             ),
             evidence=evidence,
             related_packets=sorted({p for p in (packets or []) if p is not None}),
@@ -182,6 +202,17 @@ class MalwareBeaconingDetector(BaseDetector):
         return findings
 
 
+def _cluster_bursts(ts: list[float], gap: float = 2.0) -> list[float]:
+    """Collapse packet bursts into one event so packet pacing is not mistaken for periodicity."""
+    if not ts:
+        return ts
+    out = [ts[0]]
+    for t in ts[1:]:
+        if t - out[-1] >= gap:
+            out.append(t)
+    return out
+
+
 def _external_of(ctx: CaptureContext, src_ip, dst_ip) -> Optional[str]:
     """Return the external address in a pair, if present."""
     if ctx.is_external(dst_ip):

django_project/analyzer/detectors/mitm.py

@@ -5,74 +5,177 @@ from __future__ import annotations
 from collections import defaultdict
 from typing import Iterable
 
-from ..context import CaptureContext
+from ..context import CaptureContext, is_special
 from ..findings import Finding, Category, Severity, Confidence
 from .base import BaseDetector, register
 
+_ARP_REPLY_RATE_PER_MIN = 60
+_MIN_ARP_REPLIES = 100
+
 
 @register
 class MitmDetector(BaseDetector):
     name = "MitmDetector"
-    description = "Detects ARP spoofing, duplicate IP mappings, and ARP floods."
+    description = "Detects ARP conflicts, ARP floods, rogue DHCP servers and ICMP redirects."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
 
-        # Map each IP address to MAC addresses observed in ARP responses.
-        ip_to_macs: dict[str, set[str]] = defaultdict(set)
-        first_seen: dict[str, float] = {}
-        last_seen: dict[str, float] = {}
+        # Map each IP address to MAC addresses that claimed it, in time order.
+        ip_to_macs: dict[str, dict[str, float]] = defaultdict(dict)
         packets_for_ip: dict[str, list[int]] = defaultdict(list)
 
+        def claim(ip, mac, ts, pnum):
+            if not ip or not mac or is_special(ip):
+                return
+            mac = mac.lower()
+            if mac in ("ff:ff:ff:ff:ff:ff", "00:00:00:00:00:00") or mac.startswith(("01:00:5e", "33:33")):
+                return
+            first = ip_to_macs[ip].get(mac)
+            if first is None or (ts is not None and ts < first):
+                ip_to_macs[ip][mac] = ts if ts is not None else float("inf")
+            if pnum:
+                packets_for_ip[ip].append(pnum)
+
         for ev in ctx.arp_events:
-            if ev.sender_ip and ev.sender_mac:
-                ip_to_macs[ev.sender_ip].add(ev.sender_mac)
-                if ev.timestamp is not None:
-                    first_seen[ev.sender_ip] = min(first_seen.get(ev.sender_ip, ev.timestamp), ev.timestamp)
-                    last_seen[ev.sender_ip] = max(last_seen.get(ev.sender_ip, ev.timestamp), ev.timestamp)
-                if ev.packet_number:
-                    packets_for_ip[ev.sender_ip].append(ev.packet_number)
+            if ev.opcode in (1, 2):
+                claim(ev.sender_ip, ev.sender_mac, ev.timestamp, ev.packet_number)
 
         for ip, host in ctx.hosts.items():
-            if host.macs:
-                ip_to_macs[ip].update(host.macs)
+            if host.is_private and not is_special(ip):
+                for mac in host.macs:
+                    claim(ip, mac, host.first_seen, None)
 
         for ip, macs in ip_to_macs.items():
             if len(macs) > 1:
+                ordered = sorted(macs.items(), key=lambda kv: kv[1])
+                times = [t for _, t in ordered if t != float("inf")]
                 findings.append(Finding(
                     title="Multiple MAC addresses associated with one IP",
                     category=Category.MITM,
                     severity=Severity.HIGH, confidence=Confidence.MEDIUM,
                     source_ip=ip,
                     protocol="arp",
-                    timestamp_first_seen=first_seen.get(ip),
-                    timestamp_last_seen=last_seen.get(ip),
+                    timestamp_first_seen=times[0] if times else None,
+                    timestamp_last_seen=times[-1] if times else None,
                     risk_description=(
-                        "One IP address was advertised by multiple MAC addresses, a common "
-                        "indicator of ARP spoofing or traffic interception."
+                        "One IP address was claimed by multiple MAC addresses. This happens with "
+                        "ARP spoofing and traffic interception, but also with a DHCP lease that "
+                        "moved between devices or a router fail-over."
                     ),
                     recommendation=(
-                        "Review ARP tables, enable Dynamic ARP Inspection and port security, "
-                        "and identify the conflicting host."
+                        "Identify both hosts by MAC address, review ARP tables, and enable "
+                        "Dynamic ARP Inspection or port security."
                     ),
-                    evidence={"ip": ip, "macs": sorted(macs), "dedup_key": f"arpdup:{ip}"},
+                    evidence={"ip": ip, "macs": [m for m, _ in ordered],
+                              "device": ctx.is_device(ip), "dedup_key": f"arpdup:{ip}"},
                     related_packets=sorted(packets_for_ip.get(ip, []))[:10],
                 ))
 
-        # An ARP reply flood can accompany active cache poisoning.
-        grat = [e for e in ctx.arp_events if e.opcode == 2]
-        if len(grat) >= 50:
+        # ARP reply floods: judge by rate, not by the absolute count in a long capture.
+        replies = [e for e in ctx.arp_events if e.opcode == 2]
+        requests = [e for e in ctx.arp_events if e.opcode == 1]
+        minutes = max(ctx.duration / 60.0, 1.0)
+        if len(replies) >= _MIN_ARP_REPLIES and len(replies) / minutes >= _ARP_REPLY_RATE_PER_MIN \
+                and len(replies) > 2 * max(len(requests), 1):
+            by_sender: dict[str, int] = defaultdict(int)
+            for e in replies:
+                by_sender[e.sender_mac or "?"] += 1
+            top = sorted(by_sender.items(), key=lambda kv: -kv[1])[:3]
             findings.append(Finding(
-                title="Unusually high number of ARP replies",
+                title="Unusually high ARP reply rate",
                 category=Category.MITM,
-                severity=Severity.MEDIUM, confidence=Confidence.LOW,
+                severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
                 protocol="arp",
+                timestamp_first_seen=replies[0].timestamp, timestamp_last_seen=replies[-1].timestamp,
                 risk_description=(
-                    "A large number of ARP replies may indicate active ARP cache poisoning."
+                    "Far more ARP replies than requests were seen at a high rate, which is "
+                    "characteristic of active ARP cache poisoning."
                 ),
                 recommendation="Inspect the reply source and investigate possible interception.",
-                evidence={"arp_reply_count": len(grat), "dedup_key": "arp_reply_flood"},
-                related_packets=sorted({e.packet_number for e in grat if e.packet_number})[:8],
+                evidence={"arp_replies": len(replies), "arp_requests": len(requests),
+                          "replies_per_minute": round(len(replies) / minutes, 1),
+                          "top_senders": [f"{m} ({n})" for m, n in top],
+                          "dedup_key": "arp_reply_flood"},
+                related_packets=sorted({e.packet_number for e in replies if e.packet_number})[:8],
+            ))
+
+        # Gratuitous ARP claiming an address that already belongs to another MAC.
+        seen_owner: dict[str, str] = {}
+        for e in sorted(ctx.arp_events, key=lambda e: e.timestamp or 0):
+            if not e.sender_ip or not e.sender_mac or is_special(e.sender_ip):
+                continue
+            owner = seen_owner.setdefault(e.sender_ip, e.sender_mac)
+            if e.is_gratuitous and owner != e.sender_mac:
+                findings.append(Finding(
+                    title="Gratuitous ARP overriding an existing address owner",
+                    category=Category.MITM,
+                    severity=Severity.HIGH, confidence=Confidence.MEDIUM,
+                    source_ip=e.sender_ip, protocol="arp",
+                    timestamp_first_seen=e.timestamp, timestamp_last_seen=e.timestamp,
+                    risk_description=(
+                        "A host announced an IP address that another MAC address had been using, "
+                        "which redirects traffic for that address to the announcing host."
+                    ),
+                    recommendation="Identify the announcing host and check for interception.",
+                    evidence={"ip": e.sender_ip, "previous_mac": owner, "new_mac": e.sender_mac,
+                              "dedup_key": f"garp:{e.sender_ip}:{e.sender_mac}"},
+                    related_packets=[p for p in [e.packet_number] if p],
+                ))
+
+        # More than one DHCP server answering on the segment.
+        servers: dict[str, list] = defaultdict(list)
+        for d in ctx.dhcp_events:
+            if d.message_type in ("offer", "ack") and (d.server_id or d.src_ip):
+                servers[d.server_id or d.src_ip].append(d)
+        if len(servers) > 1:
+            findings.append(Finding(
+                title="Multiple DHCP servers observed",
+                category=Category.MITM,
+                severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
+                protocol="dhcp", port=67,
+                risk_description=(
+                    "Offers or acknowledgements came from more than one DHCP server. A rogue "
+                    "DHCP server can hand out a malicious gateway or DNS resolver."
+                ),
+                recommendation="Verify every DHCP server on the segment and enable DHCP snooping.",
+                evidence={"servers": sorted(servers), "dedup_key": "dhcp_multi_server"},
+                related_packets=sorted({d.packet_number for evs in servers.values() for d in evs if d.packet_number})[:8],
+            ))
+
+        # ICMP redirects and IPv6 router advertisements from unexpected sources.
+        redirects = [s for s in ctx.service_events if s.kind == "redirect"]
+        if redirects:
+            srcs = sorted({s.src_ip for s in redirects if s.src_ip})
+            findings.append(Finding(
+                title="ICMP redirect messages observed",
+                category=Category.MITM,
+                severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
+                source_ip=srcs[0] if srcs else None, protocol="icmp",
+                timestamp_first_seen=redirects[0].timestamp, timestamp_last_seen=redirects[-1].timestamp,
+                risk_description=(
+                    "ICMP redirects change a host's routing table and are a known technique to "
+                    "insert a device into the traffic path."
+                ),
+                recommendation="Disable acceptance of ICMP redirects on hosts and verify the sender.",
+                evidence={"count": len(redirects), "sources": srcs[:5], "dedup_key": "icmp_redirect"},
+                related_packets=sorted({s.packet_number for s in redirects if s.packet_number})[:8],
+            ))
+        ras = [s for s in ctx.service_events if s.kind == "router_advertisement"]
+        ra_sources = sorted({s.src_ip for s in ras if s.src_ip})
+        if len(ra_sources) > 1:
+            findings.append(Finding(
+                title="IPv6 router advertisements from multiple sources",
+                category=Category.MITM,
+                severity=Severity.MEDIUM, confidence=Confidence.LOW,
+                protocol="icmpv6",
+                risk_description=(
+                    "More than one host advertises itself as an IPv6 router. A rogue "
+                    "advertisement can redirect all IPv6 traffic through an attacker."
+                ),
+                recommendation="Verify the legitimate router and enable RA Guard on switches.",
+                evidence={"sources": ra_sources[:5], "dedup_key": "rogue_ra"},
+                related_packets=sorted({s.packet_number for s in ras if s.packet_number})[:8],
             ))
 
         return self._tag(findings)

django_project/analyzer/detectors/plaintext_credentials.py

@@ -3,54 +3,77 @@
 from __future__ import annotations
 
 import base64
+import json
 import re
-from typing import Iterable
+from typing import Iterable, Optional
+from urllib.parse import parse_qsl, urlsplit
 
 from ..context import CaptureContext
 from ..findings import Finding, Category, Severity, Confidence
 from .base import BaseDetector, register
 
-# Common form fields used for credentials.
-_USER_FIELDS = ("user", "username", "uname", "login", "email", "uid")
-_PASS_FIELDS = ("pass", "passwd", "password", "pwd", "pw")
-_FORM_RE = re.compile(r"([A-Za-z0-9_\-\.]+)=([^&\s]+)")
+# Common form and JSON fields used for credentials.
+_USER_FIELDS = ("user", "username", "uname", "login", "email", "uid", "account", "usr", "name")
+_PASS_FIELDS = ("pass", "passwd", "password", "pwd", "pw", "passcode", "pin", "secret")
+_FORM_RE = re.compile(r"([A-Za-z0-9_\-\.\[\]]+)=([^&\s]*)")
+# "Authorization: Basic" in HTTP-like clear-text protocols (RTSP, SIP, ...).
+_BASIC_HEADER_RE = re.compile(r"(?im)^(?:Proxy-)?Authorization:\s*Basic\s+([A-Za-z0-9+/=]+)")
+
+_PROTOCOL_ALTERNATIVES = {
+    "ftp": "SFTP or FTPS", "telnet": "SSH", "pop3": "POP3S or STARTTLS",
+    "imap": "IMAPS or STARTTLS", "smtp": "SMTP with STARTTLS or submission over TLS",
+    "mqtt": "MQTT over TLS (port 8883) with client certificates",
+    "snmp": "SNMPv3 with authentication and privacy", "http": "HTTPS",
+    "rtsp": "RTSPS or a VPN", "sip": "SIP over TLS",
+}
 
 
 @register
 class PlaintextCredentialDetector(BaseDetector):
     name = "PlaintextCredentialDetector"
-    description = "Detects usernames and passwords transmitted without encryption."
+    description = "Detects usernames, passwords and community strings transmitted without encryption."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
 
         for pa in ctx.plaintext_auth:
-            if pa.username or pa.password:
-                findings.append(Finding(
-                    title=f"Clear-text credentials over {pa.protocol.upper()}",
-                    category=Category.PLAINTEXT_CREDENTIALS,
-                    severity=Severity.CRITICAL,
-                    confidence=Confidence.HIGH,
-                    source_ip=pa.src_ip, destination_ip=pa.dst_ip,
-                    protocol=pa.protocol, port=pa.dst_port,
-                    timestamp_first_seen=pa.timestamp, timestamp_last_seen=pa.timestamp,
-                    risk_description=(
-                        "A username or password was transmitted without encryption. "
-                        "A network observer could take over the account."
-                    ),
-                    recommendation=(
-                        f"Replace {pa.protocol.upper()} with an encrypted alternative "
-                        "such as SFTP, IMAPS, POP3S, or SMTP with STARTTLS, then rotate "
-                        "the affected password."
-                    ),
-                    evidence={
-                        "username": pa.username,
-                        "password": _mask(pa.password),
-                        "dedup_key": f"plain:{pa.protocol}:{pa.src_ip}:{pa.dst_ip}:{pa.username}",
-                    },
-                    related_packets=[p for p in [pa.packet_number] if p],
-                    related_streams=[s for s in [pa.stream_id] if s],
-                ))
+            if not (pa.username or pa.password):
+                continue
+            proto = pa.protocol.lower()
+            if proto == "snmp":
+                title = "Clear-text SNMP community string"
+                risk = ("An SNMPv1/v2c community string was observed. It acts as a password and "
+                        "allows anyone on the path to read or change device settings.")
+            else:
+                title = f"Clear-text credentials over {proto.upper()}"
+                risk = ("A username or password was transmitted without encryption. "
+                        "A network observer could take over the account.")
+            if pa.outcome == "failure":
+                title += " (login rejected)"
+            evidence = {
+                "username": pa.username,
+                "password": _mask(pa.password),
+                "password_length": len(pa.password) if pa.password else None,
+                "outcome": pa.outcome,
+                "dedup_key": f"plain:{proto}:{pa.src_ip}:{pa.dst_ip}:{pa.username}:{_mask(pa.password)}",
+            }
+            findings.append(Finding(
+                title=title,
+                category=Category.PLAINTEXT_CREDENTIALS,
+                severity=Severity.CRITICAL if proto != "snmp" else Severity.HIGH,
+                confidence=Confidence.HIGH,
+                source_ip=pa.src_ip, destination_ip=pa.dst_ip,
+                protocol=proto, port=pa.dst_port,
+                timestamp_first_seen=pa.timestamp, timestamp_last_seen=pa.timestamp,
+                risk_description=risk,
+                recommendation=(
+                    f"Replace {proto.upper()} with {_PROTOCOL_ALTERNATIVES.get(proto, 'an encrypted alternative')}, "
+                    "then rotate the affected credential."
+                ),
+                evidence=evidence,
+                related_packets=[p for p in [pa.packet_number] if p],
+                related_streams=[s for s in [pa.stream_id] if s is not None],
+            ))
 
         for req in ctx.http_requests:
             if req.is_tls:
@@ -58,59 +81,139 @@ class PlaintextCredentialDetector(BaseDetector):
             auth = (req.authorization or "").strip()
             if auth.lower().startswith("basic "):
                 user, pwd = _decode_basic(auth[6:])
+                findings.append(self._basic_finding(
+                    req.src_ip, req.dst_ip, req.dst_port, req.timestamp, req.packet_number,
+                    user, pwd, host=req.host, uri=req.uri, protocol="http"))
+
+            # Login forms: URL-encoded or JSON bodies, and credentials in query strings.
+            creds = None
+            if req.method and req.method.upper() in ("POST", "PUT", "PATCH") and req.body:
+                creds = _credentials_in_body(req.body, req.content_type)
+            if creds is None and req.uri and "?" in req.uri:
+                creds = _credentials_in_query(req.uri)
+            if creds:
+                user_val, pass_val, where = creds
                 findings.append(Finding(
-                    title="Clear-text HTTP Basic authentication",
+                    title="Login form submitted over clear-text HTTP",
                     category=Category.PLAINTEXT_CREDENTIALS,
                     severity=Severity.CRITICAL,
-                    confidence=Confidence.HIGH,
+                    confidence=Confidence.HIGH if user_val else Confidence.MEDIUM,
                     source_ip=req.src_ip, destination_ip=req.dst_ip,
                     protocol="http", port=req.dst_port,
                     timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
                     risk_description=(
-                        "HTTP Basic authentication sends credentials with Base64 encoding, "
-                        "not encryption, over clear-text HTTP."
+                        f"A login was submitted in a clear-text HTTP request ({where}). "
+                        "The credentials are visible to network observers."
                     ),
-                    recommendation="Require HTTPS and rotate the affected password.",
+                    recommendation="Serve the login only over HTTPS and rotate the password.",
                     evidence={
                         "host": req.host, "uri": req.uri,
-                        "username": user, "password": _mask(pwd),
-                        "dedup_key": f"basic:{req.src_ip}:{req.dst_ip}:{user}",
+                        "username": user_val, "password": _mask(pass_val),
+                        "location": where,
+                        "dedup_key": f"form:{req.src_ip}:{req.dst_ip}:{req.uri}:{user_val}",
                     },
                     related_packets=[p for p in [req.packet_number] if p],
                 ))
 
-            # Detect likely login forms in clear-text HTTP request bodies.
-            if req.method and req.method.upper() == "POST" and req.body:
-                fields = dict(_FORM_RE.findall(req.body))
-                lowered = {k.lower(): v for k, v in fields.items()}
-                has_user = any(k in lowered for k in _USER_FIELDS)
-                has_pass = any(k in lowered for k in _PASS_FIELDS)
-                if has_pass and (has_user or len(fields) <= 4):
-                    user_val = next((lowered[k] for k in _USER_FIELDS if k in lowered), None)
-                    pass_val = next((lowered[k] for k in _PASS_FIELDS if k in lowered), None)
-                    findings.append(Finding(
-                        title="Login form submitted over clear-text HTTP",
-                        category=Category.PLAINTEXT_CREDENTIALS,
-                        severity=Severity.CRITICAL,
-                        confidence=Confidence.MEDIUM,
-                        source_ip=req.src_ip, destination_ip=req.dst_ip,
-                        protocol="http", port=req.dst_port,
-                        timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
-                        risk_description=(
-                            "A login form was submitted in a clear-text HTTP request. "
-                            "The credentials are visible to network observers."
-                        ),
-                        recommendation="Serve the form only over HTTPS and rotate the password.",
-                        evidence={
-                            "host": req.host, "uri": req.uri,
-                            "username": user_val, "password": _mask(pass_val),
-                            "dedup_key": f"form:{req.src_ip}:{req.dst_ip}:{user_val}",
-                        },
-                        related_packets=[p for p in [req.packet_number] if p],
-                    ))
+        # Basic authentication in other clear-text protocols with HTTP-like headers.
+        seen_basic: set[str] = set()
+        for payload in ctx.text_payloads:
+            proto = (payload.get("protocol") or "").lower()
+            if proto == "http":
+                continue  # Covered via parsed HTTP requests.
+            for token in _BASIC_HEADER_RE.findall(payload.get("text") or ""):
+                key = f"{proto}:{payload.get('src_ip')}:{payload.get('dst_ip')}:{token[:32]}"
+                if key in seen_basic:
+                    continue
+                seen_basic.add(key)
+                user, pwd = _decode_basic(token)
+                findings.append(self._basic_finding(
+                    payload.get("src_ip"), payload.get("dst_ip"), None, payload.get("timestamp"),
+                    payload.get("packet_number"), user, pwd, protocol=proto or "tcp"))
 
         return self._tag(findings)
 
+    def _basic_finding(self, src, dst, port, ts, pkt, user, pwd, *, protocol, host=None, uri=None) -> Finding:
+        return Finding(
+            title=f"Clear-text Basic authentication over {protocol.upper()}",
+            category=Category.PLAINTEXT_CREDENTIALS,
+            severity=Severity.CRITICAL,
+            confidence=Confidence.HIGH,
+            source_ip=src, destination_ip=dst,
+            protocol=protocol, port=port,
+            timestamp_first_seen=ts, timestamp_last_seen=ts,
+            risk_description=(
+                "Basic authentication only Base64-encodes the credentials; over an unencrypted "
+                "transport they are readable by anyone on the path."
+            ),
+            recommendation=f"Require {_PROTOCOL_ALTERNATIVES.get(protocol, 'an encrypted transport')} and rotate the affected password.",
+            evidence={
+                "host": host, "uri": uri,
+                "username": user, "password": _mask(pwd),
+                "dedup_key": f"basic:{protocol}:{src}:{dst}:{user}",
+            },
+            related_packets=[p for p in [pkt] if p],
+        )
+
+
+def _credentials_in_body(body: str, content_type: Optional[str]):
+    text = body.strip()
+    ct = (content_type or "").lower()
+    # JSON bodies used by most device web interfaces and mobile apps.
+    if text.startswith("{") or "json" in ct:
+        try:
+            data = json.loads(text)
+        except ValueError:
+            data = None
+        if isinstance(data, dict):
+            flat = _flatten_json(data)
+            found = _pick(flat)
+            if found:
+                return found[0], found[1], "JSON body"
+    if "=" in text and (not text.startswith("{")):
+        fields = {k.lower(): v for k, v in _FORM_RE.findall(text)}
+        found = _pick(fields)
+        if found:
+            return found[0], found[1], "form body"
+    return None
+
+
+def _credentials_in_query(uri: str):
+    try:
+        query = urlsplit(uri).query
+    except ValueError:
+        return None
+    if not query:
+        return None
+    fields = {k.lower(): v for k, v in parse_qsl(query, keep_blank_values=True)}
+    found = _pick(fields)
+    if found:
+        return found[0], found[1], "URL query string"
+    return None
+
+
+def _flatten_json(data, prefix: str = "", out: Optional[dict] = None) -> dict:
+    out = out if out is not None else {}
+    if isinstance(data, dict):
+        for k, v in data.items():
+            _flatten_json(v, str(k).lower(), out)
+    elif isinstance(data, list):
+        for v in data[:20]:
+            _flatten_json(v, prefix, out)
+    else:
+        if prefix and prefix not in out:
+            out[prefix] = "" if data is None else str(data)
+    return out
+
+
+def _pick(fields: dict):
+    """Return (username, password) when a password-like field carries a value."""
+    pass_key = next((k for k in fields if k in _PASS_FIELDS or k.endswith("password") or k.endswith("passwd")), None)
+    if pass_key is None or not fields.get(pass_key):
+        return None
+    user_key = next((k for k in fields if k in _USER_FIELDS or k.endswith("username") or k.endswith("user")), None)
+    return (fields.get(user_key) if user_key else None), fields[pass_key]
+
 
 def _decode_basic(b64: str):
     try:

django_project/analyzer/detectors/tls_weakness.py

@@ -1,93 +1,160 @@
-"""Detect weak, obsolete, or suspicious TLS connections."""
+"""Detect weak, obsolete, or suspicious TLS configurations.
+
+Negotiated parameters (ServerHello) are judged strictly; what a client merely
+offers is reported at lower severity.
+"""
 
 from __future__ import annotations
 
+from collections import defaultdict
 from typing import Iterable
 
-from ..context import CaptureContext
+from ..context import CaptureContext, TlsSession
 from ..findings import Finding, Category, Severity, Confidence
+from .. import tls_tables
 from .base import BaseDetector, register
 
 _WEAK_VERSIONS = {"SSL 2.0", "SSL 3.0", "TLS 1.0", "TLS 1.1"}
-# Weak cipher markers matched against normalized suite names.
-_WEAK_CIPHER_MARKERS = ("RC4", "NULL", "EXPORT", "_DES_", "DES_CBC", "3DES", "MD5", "ANON")
+_FAILURE_ALERTS = {"handshake_failure", "bad_certificate", "unknown_ca", "certificate_expired",
+                   "certificate_unknown", "unsupported_certificate", "protocol_version",
+                   "insufficient_security", "certificate_revoked", "decrypt_error"}
 
 
 @register
 class TlsWeaknessDetector(BaseDetector):
     name = "TlsWeaknessDetector"
-    description = "Detects obsolete TLS versions, weak ciphers, and certificate anomalies."
+    description = "Detects obsolete TLS versions, weak or legacy cipher suites, certificate problems and handshake failures."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
         seen: set[str] = set()
 
+        def emit(key, **kw):
+            if key in seen:
+                return
+            seen.add(key)
+            kw.setdefault("category", Category.TLS_WEAKNESS)
+            kw.setdefault("protocol", "tls")
+            kw["evidence"] = dict(kw.get("evidence") or {}, dedup_key=key)
+            findings.append(Finding(**kw))
+
         for s in ctx.tls_sessions:
             target = s.sni or s.dst_ip
-            base_pkt = [p for p in [s.packet_number] if p]
+            pkts = [p for p in [s.packet_number] if p]
+            common = dict(source_ip=s.src_ip, destination_ip=s.dst_ip, port=s.dst_port,
+                          timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
+                          related_packets=pkts)
 
-            if s.version and s.version in _WEAK_VERSIONS:
-                key = f"tlsver:{target}:{s.version}"
-                if key not in seen:
-                    seen.add(key)
-                    findings.append(Finding(
-                        title=f"Obsolete TLS version: {s.version}",
-                        category=Category.TLS_WEAKNESS,
-                        severity=Severity.MEDIUM, confidence=Confidence.HIGH,
-                        source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="tls", port=s.dst_port,
-                        timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
-                        risk_description=(
-                            "An obsolete TLS or SSL version is in use and is vulnerable to "
-                            "known protocol attacks."
-                        ),
-                        recommendation="Require at least TLS 1.2 and prefer TLS 1.3.",
-                        evidence={"version": s.version, "target": target, "dedup_key": key},
-                        related_packets=base_pkt,
-                    ))
+            # 1) Negotiated version and cipher (ServerHello seen).
+            if s.server_hello_seen and s.version in _WEAK_VERSIONS:
+                emit(f"tlsver:{s.src_ip}:{target}:{s.version}",
+                     title=f"Obsolete TLS version negotiated: {s.version}",
+                     severity=Severity.HIGH, confidence=Confidence.HIGH,
+                     risk_description=("The connection was established with an obsolete protocol version "
+                                       "that is vulnerable to known attacks."),
+                     recommendation="Require at least TLS 1.2 and prefer TLS 1.3 on both sides.",
+                     evidence={"version": s.version, "target": target, "cipher": s.cipher}, **common)
+            if s.server_hello_seen and s.cipher:
+                weak = tls_tables.cipher_is_weak(s.cipher)
+                legacy = tls_tables.cipher_is_legacy(s.cipher)
+                if weak:
+                    emit(f"tlscipher:{s.src_ip}:{target}:{s.cipher}",
+                         title=f"Weak TLS cipher suite negotiated ({weak})",
+                         severity=Severity.HIGH, confidence=Confidence.HIGH,
+                         risk_description=f"The negotiated cipher suite {s.cipher} is considered broken: {weak}.",
+                         recommendation="Allow only AEAD suites (AES-GCM, ChaCha20-Poly1305) with ECDHE key exchange.",
+                         evidence={"cipher": s.cipher, "target": target, "version": s.version}, **common)
+                elif legacy:
+                    emit(f"tlslegacy:{s.src_ip}:{target}:{s.cipher}",
+                         title="Legacy TLS cipher suite negotiated",
+                         severity=Severity.LOW, confidence=Confidence.HIGH,
+                         risk_description=f"The negotiated cipher suite {s.cipher} is outdated: {legacy}.",
+                         recommendation="Prefer TLS 1.3 or TLS 1.2 with ECDHE and AEAD cipher suites.",
+                         evidence={"cipher": s.cipher, "target": target, "version": s.version}, **common)
 
-            if s.cipher:
-                up = str(s.cipher).upper()
-                if any(m in up for m in _WEAK_CIPHER_MARKERS):
-                    key = f"tlscipher:{target}:{s.cipher}"
-                    if key not in seen:
-                        seen.add(key)
-                        findings.append(Finding(
-                            title="Weak TLS cipher suite",
-                            category=Category.TLS_WEAKNESS,
-                            severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
-                            source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="tls", port=s.dst_port,
-                            timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
-                            risk_description=(
-                                "A weak or obsolete cipher suite reduces transport protection."
-                            ),
-                            recommendation="Allow only modern AEAD ciphers such as AES-GCM or ChaCha20.",
-                            evidence={"cipher": s.cipher, "target": target, "dedup_key": key},
-                            related_packets=base_pkt,
-                        ))
+            # 2) Client capabilities (what the device would accept).
+            if s.client_hello_seen:
+                if s.client_version in _WEAK_VERSIONS:
+                    emit(f"clientver:{s.src_ip}:{s.client_version}",
+                         title=f"Client supports at most {s.client_version}",
+                         severity=Severity.MEDIUM, confidence=Confidence.HIGH,
+                         risk_description=("The client's highest offered protocol version is obsolete, so every "
+                                           "connection it makes uses a version with known weaknesses."),
+                         recommendation="Update the device TLS stack to support TLS 1.2 or TLS 1.3.",
+                         evidence={"client_max_version": s.client_version, "offered": s.client_offered_versions,
+                                   "target": target}, **common)
+                elif s.client_version == "TLS 1.2" and ctx.is_device(s.src_ip):
+                    emit(f"clientno13:{s.src_ip}",
+                         title="Device does not offer TLS 1.3",
+                         severity=Severity.INFO, confidence=Confidence.HIGH,
+                         risk_description="The device negotiates at most TLS 1.2. This is acceptable today but limits forward compatibility.",
+                         recommendation="Consider adding TLS 1.3 support in a future firmware.",
+                         evidence={"client_max_version": s.client_version, "target": target}, **common)
+                weak_offers = [(c, tls_tables.cipher_is_weak(c)) for c in s.client_offered_ciphers]
+                weak_offers = [(c, r) for c, r in weak_offers if r]
+                if weak_offers:
+                    emit(f"clientweak:{s.src_ip}",
+                         title="Client offers broken cipher suites",
+                         severity=Severity.MEDIUM if ctx.is_device(s.src_ip) else Severity.LOW,
+                         confidence=Confidence.HIGH,
+                         risk_description=("The client advertises cipher suites with known weaknesses. A "
+                                           "downgrade or a misconfigured server could select one of them."),
+                         recommendation="Remove obsolete cipher suites from the client's TLS configuration.",
+                         evidence={"weak_ciphers": [f"{c} ({r})" for c, r in weak_offers][:8],
+                                   "offered_total": len(s.client_offered_ciphers), "target": target}, **common)
+                if ctx.is_external(s.dst_ip) and not s.sni and s.transport == "tcp":
+                    emit(f"nosni:{s.src_ip}:{s.dst_ip}",
+                         title="TLS connection to an external host without SNI",
+                         severity=Severity.LOW, confidence=Confidence.MEDIUM,
+                         risk_description=("The client did not send a server name. Connections by raw IP address "
+                                           "often skip hostname verification of the server certificate."),
+                         recommendation="Connect by host name and validate the certificate against it.",
+                         evidence={"target": s.dst_ip, "port": s.dst_port}, **common)
 
-            if s.cert_self_signed or s.cert_expired:
-                problem = []
+            # 3) Certificate properties (visible in TLS 1.2 and earlier).
+            if s.cert_seen:
+                problems = []
                 if s.cert_self_signed:
-                    problem.append("self-signed")
+                    problems.append("self-signed")
                 if s.cert_expired:
-                    problem.append("expired")
-                key = f"tlscert:{target}:{'-'.join(problem)}"
-                if key not in seen:
-                    seen.add(key)
-                    findings.append(Finding(
-                        title=f"Suspicious TLS certificate ({', '.join(problem)})",
-                        category=Category.TLS_WEAKNESS,
-                        severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
-                        source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="tls", port=s.dst_port,
-                        timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
-                        risk_description=(
-                            "A self-signed or expired certificate may indicate endpoint "
-                            "misconfiguration or TLS interception."
-                        ),
-                        recommendation="Review the certificate chain and deploy a valid CA certificate.",
-                        evidence={"subject_cn": s.cert_subject_cn, "issuer_cn": s.cert_issuer_cn,
-                                  "not_after": s.cert_not_after, "dedup_key": key},
-                        related_packets=base_pkt,
-                    ))
+                    problems.append("expired")
+                sig = (s.cert_sig_algorithm or "").lower()
+                if sig in ("sha1", "md5", "md2"):
+                    problems.append(f"{sig.upper()} signature")
+                if (s.cert_key_type or "").startswith("RSA") and s.cert_key_bits and s.cert_key_bits < 2048:
+                    problems.append(f"RSA {s.cert_key_bits}-bit key")
+                if problems:
+                    sev = Severity.MEDIUM
+                    if s.cert_expired or "self-signed" in problems and ctx.is_external(s.dst_ip):
+                        sev = Severity.HIGH if ctx.is_external(s.dst_ip) else Severity.MEDIUM
+                    emit(f"tlscert:{target}:{'-'.join(problems)}",
+                         title=f"Weak or untrusted server certificate ({', '.join(problems)})",
+                         severity=sev, confidence=Confidence.HIGH,
+                         risk_description=("The server certificate has properties that prevent proper validation "
+                                           "or rely on weak cryptography. Clients that still connect probably do not "
+                                           "validate certificates, which enables interception."),
+                         recommendation="Deploy a certificate from a trusted CA with SHA-256 and at least RSA 2048 or ECDSA P-256, and make clients validate it.",
+                         evidence={"subject_cn": s.cert_subject_cn, "issuer_cn": s.cert_issuer_cn,
+                                   "not_after": s.cert_not_after, "signature": s.cert_sig_algorithm,
+                                   "key": f"{s.cert_key_type} {s.cert_key_bits}" if s.cert_key_type else None,
+                                   "target": target}, **common)
+                if s.client_cert_seen and ctx.is_device(s.src_ip):
+                    emit(f"mtls:{s.src_ip}:{target}",
+                         title="Device authenticates with a client certificate (mutual TLS)",
+                         severity=Severity.INFO, confidence=Confidence.HIGH,
+                         risk_description="The device presented its own certificate during the handshake. This is a strong authentication mechanism and worth documenting.",
+                         recommendation="No action required; ensure the device key is stored in protected hardware where possible.",
+                         evidence={"target": target, "version": s.version, "cipher": s.cipher}, **common)
+
+            # 4) Handshake failures / alerts.
+            failed = [a for a in s.alerts if a in _FAILURE_ALERTS]
+            if failed:
+                emit(f"tlsalert:{s.src_ip}:{target}:{failed[0]}",
+                     title=f"TLS handshake failure ({failed[0]})",
+                     severity=Severity.LOW, confidence=Confidence.MEDIUM,
+                     risk_description=("A TLS alert aborted the handshake. Repeated failures can indicate "
+                                       "incompatible configurations, certificate problems or interception attempts."),
+                     recommendation="Check certificate validity and the supported cipher/version overlap between client and server.",
+                     evidence={"alerts": s.alerts, "target": target}, **common)
 
         return self._tag(findings)

django_project/analyzer/detectors/tokens_secrets.py

@@ -1,4 +1,4 @@
-"""Detect session cookies, bearer tokens, and API keys."""
+"""Detect session cookies, bearer tokens, API keys and key material in clear text."""
 
 from __future__ import annotations
 
@@ -10,23 +10,34 @@ from ..findings import Finding, Category, Severity, Confidence
 from .base import BaseDetector, register
 
 # JWTs consist of three base64url segments and typically start with ``eyJ``.
-_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+")
-_BEARER_RE = re.compile(r"\bBearer\s+([A-Za-z0-9_\-\.=]+)", re.IGNORECASE)
+_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}")
+_BEARER_RE = re.compile(r"\bBearer\s+([A-Za-z0-9_\-\.=]{16,})", re.IGNORECASE)
 # Generic API key patterns in URLs, bodies, and headers.
 _APIKEY_RE = re.compile(
-    r"(?i)(api[_\-]?key|apikey|access[_\-]?token|client[_\-]?secret|x[_\-]api[_\-]key)"
-    r"[=:\"']?\s*([A-Za-z0-9_\-]{16,})"
+    r"(?i)\b(api[_\-]?key|apikey|access[_\-]?token|auth[_\-]?token|client[_\-]?secret|"
+    r"x[_\-]api[_\-]key|x[_\-]auth[_\-]token|x[_\-]access[_\-]token|private[_\-]?key|app[_\-]?secret)"
+    r"\"?\s*[=:]\s*\"?([A-Za-z0-9_\-\.]{16,})"
 )
 _SESSION_COOKIE_RE = re.compile(
-    r"(?i)\b(sessionid|session|sid|phpsessid|jsessionid|asp\.net_sessionid|auth|token)="
+    r"(?i)\b(sessionid|session|sid|phpsessid|jsessionid|asp\.net_sessionid|auth|token|"
+    r"remember_token|access_token|connect\.sid)="
     r"([A-Za-z0-9%_\-\.]{8,})"
 )
+_PRIVATE_KEY_RE = re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")
+# Well-known provider key shapes.
+_PROVIDER_KEYS = (
+    ("aws_access_key", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b")),
+    ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b")),
+    ("slack_token", re.compile(r"\bxox[abpr]-[A-Za-z0-9\-]{10,}\b")),
+    ("google_api_key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")),
+    ("stripe_key", re.compile(r"\b[sr]k_(?:live|test)_[A-Za-z0-9]{16,}\b")),
+)
 
 
 @register
 class TokenSecretDetector(BaseDetector):
     name = "TokenSecretDetector"
-    description = "Detects JWTs, bearer tokens, API keys, and session cookies."
+    description = "Detects JWTs, bearer tokens, API keys, private keys and session cookies."
 
     def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
         findings: list[Finding] = []
@@ -34,30 +45,50 @@ class TokenSecretDetector(BaseDetector):
 
         for payload in ctx.text_payloads:
             text = payload.get("text") or ""
-            over_http = payload.get("protocol") == "http"
+            proto = (payload.get("protocol") or "").lower()
+            over_http = proto == "http"
 
             for m in _JWT_RE.findall(text):
                 findings.append(self._mk(
                     payload, "Clear-text JWT (JSON Web Token)",
                     "A JWT was transmitted in clear text and may grant passwordless access "
                     "until it expires.",
-                    "Revoke and rotate the token, shorten its lifetime, and require HTTPS.",
+                    "Revoke and rotate the token, shorten its lifetime, and require TLS.",
                     secret=m, conf=Confidence.HIGH, seen=seen, kind="jwt"))
 
             for m in _BEARER_RE.findall(text):
                 findings.append(self._mk(
                     payload, "Clear-text bearer or OAuth token",
                     "A bearer token was transmitted and may grant access without a password.",
-                    "Rotate the token, shorten its lifetime, and require HTTPS.",
+                    "Rotate the token, shorten its lifetime, and require TLS.",
                     secret=m, conf=Confidence.HIGH, seen=seen, kind="bearer"))
 
             for _, m in _APIKEY_RE.findall(text):
                 findings.append(self._mk(
                     payload, "Clear-text API key or client secret",
                     "An API key or client secret was transmitted in clear text.",
-                    "Rotate the secret, store it securely, and require HTTPS.",
+                    "Rotate the secret, store it securely, and require TLS.",
                     secret=m, conf=Confidence.MEDIUM, seen=seen, kind="apikey"))
 
+            for kind, rx in _PROVIDER_KEYS:
+                for m in rx.findall(text):
+                    value = m if isinstance(m, str) else m[0]
+                    findings.append(self._mk(
+                        payload, f"Clear-text provider credential ({kind.replace('_', ' ')})",
+                        "A cloud or service provider credential was transmitted in clear text.",
+                        "Revoke the credential at the provider and issue a new one.",
+                        secret=value, conf=Confidence.HIGH, seen=seen, kind=kind,
+                        severity=Severity.CRITICAL))
+
+            if _PRIVATE_KEY_RE.search(text):
+                findings.append(self._mk(
+                    payload, "Private key material transmitted in clear text",
+                    "A PEM-encoded private key was observed in an unencrypted flow.",
+                    "Treat the key as compromised: revoke it, generate a new key pair and "
+                    "provision keys only over an authenticated, encrypted channel.",
+                    secret="PRIVATE KEY", conf=Confidence.HIGH, seen=seen, kind="private_key",
+                    severity=Severity.CRITICAL))
+
             # Report session cookies only when the transport is unencrypted.
             if over_http:
                 for _, m in _SESSION_COOKIE_RE.findall(text):
@@ -69,7 +100,8 @@ class TokenSecretDetector(BaseDetector):
 
         return self._tag([f for f in findings if f is not None])
 
-    def _mk(self, payload, title, risk, rec, *, secret, conf, seen, kind):
+    def _mk(self, payload, title, risk, rec, *, secret, conf, seen, kind,
+            severity: Severity = Severity.HIGH):
         key = f"{kind}:{secret[:24]}"
         if key in seen:
             return None
@@ -77,7 +109,7 @@ class TokenSecretDetector(BaseDetector):
         return Finding(
             title=title,
             category=Category.TOKENS_SECRETS,
-            severity=Severity.HIGH, confidence=conf,
+            severity=severity, confidence=conf,
             source_ip=payload.get("src_ip"), destination_ip=payload.get("dst_ip"),
             protocol=payload.get("protocol"),
             timestamp_first_seen=payload.get("timestamp"),

django_project/analyzer/engine.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import logging
 import time
 from dataclasses import dataclass, field
 from typing import Optional
@@ -11,6 +12,9 @@ from .findings import Finding
 from . import scoring
 from . import provenance
 from . import detectors as detectors_pkg
+from . import inventory as inventory_mod
+
+logger = logging.getLogger("analyzer.engine")
 
 
 @dataclass
@@ -18,12 +22,15 @@ class AnalysisResult:
     findings: list[Finding] = field(default_factory=list)
     summary: dict = field(default_factory=dict)
     meta: dict = field(default_factory=dict)
+    # Complete observation inventory (hosts, endpoints, protocols, TLS posture, ...).
+    inventory: dict = field(default_factory=dict)
 
     def to_dict(self) -> dict:
         return {
             "meta": self.meta,
             "summary": _summary_to_dict(self.summary),
             "findings": [f.to_dict() for f in self.findings],
+            "inventory": self.inventory,
         }
 
 
@@ -36,49 +43,93 @@ class AnalysisEngine:
         started = time.time()
         raw: list[Finding] = []
         errors: dict[str, str] = {}
+        timings: dict[str, float] = {}
+        logger.info("Analysis start: %d detectors, %d packets, device_ip=%s",
+                    len(self.detectors), ctx.packet_count, ctx.device_ip)
 
         for det in self.detectors:
+            t0 = time.time()
             try:
-                raw.extend(list(det.detect(ctx)))
+                found = list(det.detect(ctx))
+                raw.extend(found)
+                logger.debug("%s: %d findings in %.3fs", det.name, len(found), time.time() - t0)
             except Exception as exc:  # One detector must not abort the full analysis.
                 errors[det.name] = repr(exc)
+                logger.exception("%s failed: %r", det.name, exc)
+            timings[det.name] = round(time.time() - t0, 3)
 
         deduped = scoring.dedupe(raw)
+        # Attribute findings to the device under test before prioritizing.
+        for f in deduped:
+            _attribute_device(f, ctx)
         ordered = scoring.sort_by_priority(deduped)
         # Attach packet-level provenance before the summary is generated.
         capture_file = _basename(ctx.path)
         for f in ordered:
             _enrich_provenance(f, capture_file)
         summary = scoring.summarize(ordered)
+        try:
+            inventory = inventory_mod.build_inventory(ctx)
+        except Exception as exc:  # The inventory must never break the analysis.
+            logger.exception("Inventory build failed: %r", exc)
+            inventory = {"error": repr(exc)}
+        logger.info("Analysis done: %d raw -> %d findings (%s) in %.2fs; detector errors: %s",
+                    len(raw), len(ordered), summary.get("by_severity"), time.time() - started,
+                    errors or "none")
 
         return AnalysisResult(
             findings=ordered,
             summary=summary,
+            inventory=inventory,
             meta={
                 "capture_path": ctx.path,
                 "packet_count": ctx.packet_count,
                 "duration_seconds": round(ctx.duration, 3),
+                "capture_start": ctx.start_time,
+                "capture_end": ctx.end_time,
+                "device_ip": ctx.device_ip,
+                "device_macs": sorted(ctx.device_macs),
+                "extractor": ctx.extractor,
+                "coverage_notes": list(ctx.coverage_notes),
                 "detectors_run": [d.name for d in self.detectors],
                 "detector_errors": errors,
+                "detector_timings": timings,
                 "analysis_time_seconds": round(time.time() - started, 3),
                 "raw_finding_count": len(raw),
                 "finding_count": len(ordered),
+                # A partial pass must never be mistaken for a complete one.
+                "truncated": bool(getattr(ctx, "truncated", False)),
+                "truncation_reason": getattr(ctx, "truncation_reason", None),
             },
         )
 
-    def analyze_pcap(self, path: str, packet_limit: Optional[int] = None) -> AnalysisResult:
-        # Keep context-only analysis usable without pyshark or tshark.
-        from .extract_pyshark import build_context_from_pcap
-        ctx = build_context_from_pcap(path, packet_limit=packet_limit)
+    def analyze_pcap(self, path: str, packet_limit: Optional[int] = None,
+                     device_ip: Optional[str] = None) -> AnalysisResult:
+        # Keep context-only analysis usable without tshark.
+        from .extract_tshark import build_context_from_pcap
+        ctx = build_context_from_pcap(path, packet_limit=packet_limit, device_ip=device_ip)
         return self.analyze_context(ctx)
 
 
 def result_from_dict(data: dict) -> AnalysisResult:
-    """Reconstruct a result and recalculate its summary from stored findings."""
+    """Rebuild a result from stored JSON; the summary is recalculated."""
     findings = [Finding.from_dict(f) for f in data.get("findings", [])]
     summary = scoring.summarize(findings)
     meta = data.get("meta", {})
-    return AnalysisResult(findings=findings, summary=summary, meta=meta)
+    return AnalysisResult(findings=findings, summary=summary, meta=meta,
+                          inventory=data.get("inventory") or {})
+
+
+def _attribute_device(f: Finding, ctx: CaptureContext) -> None:
+    """Set affected_device when the finding touches the device under test."""
+    if ctx.device_ip is None:
+        return
+    involved = ctx.involves_device(f.source_ip, f.destination_ip)
+    if involved is None:
+        return
+    f.evidence["device_related"] = bool(involved)
+    if involved and not f.affected_device:
+        f.affected_device = ctx.device_ip
 
 
 def _summary_to_dict(summary: dict) -> dict:

django_project/analyzer/extract_pyshark.py

@@ -1,763 +0,0 @@
-"""Build a normalized capture context with tshark through pyshark.
-
-pyshark is loaded lazily, and individual packet handlers are isolated so one
-unexpected packet cannot abort the complete analysis.
-"""
-
-from __future__ import annotations
-
-import base64
-import os
-from typing import Optional
-
-from .context import (
-    CaptureContext,
-    Conversation,
-    DnsQuery,
-    HttpRequest,
-    HttpResponse,
-    TlsSession,
-    ArpEvent,
-    PlaintextAuth,
-    AuthMaterial,
-    FileTransfer,
-)
-from . import crackformats
-
-# Known clear-text protocols used by extraction heuristics.
-CLEARTEXT_PORTS = {
-    21: "ftp",
-    23: "telnet",
-    25: "smtp",
-    80: "http",
-    110: "pop3",
-    143: "imap",
-    389: "ldap",
-    161: "snmp",
-    69: "tftp",
-}
-
-
-def build_context_from_pcap(path: str, packet_limit: Optional[int] = None) -> CaptureContext:
-    """Read a capture and return its normalized context."""
-    if not os.path.exists(path):
-        raise FileNotFoundError(f"Capture not found: {path}")
-
-    try:
-        import pyshark  # lazy import
-    except ImportError as exc:  # pragma: no cover
-        raise RuntimeError(
-            "pyshark or tshark is unavailable. Install tshark or use the backend container."
-        ) from exc
-
-    ctx = CaptureContext(path=path)
-    cap = pyshark.FileCapture(path, keep_packets=False)
-
-    conv_index: dict[tuple, Conversation] = {}
-    # Transient state correlates NTLM and SMTP authentication across packets.
-    state: dict = {"ntlm_chal": {}, "smtp_auth": {}}
-    n = 0
-    try:
-        for pkt in cap:
-            n += 1
-            if packet_limit and n > packet_limit:
-                break
-            try:
-                _handle_packet(ctx, pkt, conv_index, state)
-            except Exception:
-                # Skip isolated malformed or unexpected packets.
-                continue
-    except Exception as exc:
-        if exc.__class__.__name__ == "TSharkNotFoundException":
-            raise RuntimeError(
-                "tshark was not found. Install Wireshark/tshark or use the backend container."
-            ) from exc
-        raise
-    finally:
-        try:
-            cap.close()
-        except Exception:
-            pass
-
-    ctx.packet_count = n
-    ctx.conversations = list(conv_index.values())
-    return ctx
-
-
-# --------------------------------------------------------------------------- #
-# Packet handlers
-# --------------------------------------------------------------------------- #
-
-def _ts(pkt) -> Optional[float]:
-    try:
-        return float(pkt.sniff_timestamp)
-    except Exception:
-        return None
-
-
-def _num(pkt) -> Optional[int]:
-    try:
-        return int(pkt.number)
-    except Exception:
-        return None
-
-
-def _handle_packet(ctx: CaptureContext, pkt, conv_index: dict, state: Optional[dict] = None) -> None:
-    if state is None:
-        state = {"ntlm_chal": {}, "smtp_auth": {}}
-    ts = _ts(pkt)
-    if ts is not None:
-        ctx.start_time = ts if ctx.start_time is None else min(ctx.start_time, ts)
-        ctx.end_time = ts if ctx.end_time is None else max(ctx.end_time, ts)
-
-    # Count packets by the highest decoded protocol layer.
-    try:
-        top = pkt.highest_layer
-        ctx.protocol_stats[top] = ctx.protocol_stats.get(top, 0) + 1
-    except Exception:
-        pass
-
-    layers = {l.layer_name for l in pkt.layers}
-
-    src_ip = dst_ip = None
-    length = 0
-    try:
-        length = int(pkt.length)
-    except Exception:
-        pass
-
-    if "ip" in layers:
-        try:
-            src_ip = pkt.ip.src
-            dst_ip = pkt.ip.dst
-        except Exception:
-            pass
-    elif "ipv6" in layers:
-        try:
-            src_ip = pkt.ipv6.src
-            dst_ip = pkt.ipv6.dst
-        except Exception:
-            pass
-
-    # MAC-Adressen / Hosts
-    src_mac = dst_mac = None
-    if "eth" in layers:
-        try:
-            src_mac, dst_mac = pkt.eth.src, pkt.eth.dst
-        except Exception:
-            pass
-
-    src_port = dst_port = None
-    transport = None
-    if "tcp" in layers:
-        transport = "tcp"
-        try:
-            src_port = int(pkt.tcp.srcport)
-            dst_port = int(pkt.tcp.dstport)
-        except Exception:
-            pass
-    elif "udp" in layers:
-        transport = "udp"
-        try:
-            src_port = int(pkt.udp.srcport)
-            dst_port = int(pkt.udp.dstport)
-        except Exception:
-            pass
-
-    if src_ip:
-        h = ctx.host(src_ip)
-        h.bytes_sent += length
-        if src_mac:
-            h.macs.add(src_mac)
-        if dst_port:
-            h.ports_contacted.add(dst_port)
-    if dst_ip:
-        h = ctx.host(dst_ip)
-        h.bytes_received += length
-        if dst_port:
-            h.ports_offered.add(dst_port)
-
-    # Conversation aggregieren
-    if src_ip and dst_ip and transport:
-        key = tuple(sorted([(src_ip, src_port), (dst_ip, dst_port)]) + [transport])
-        conv = conv_index.get(key)
-        if conv is None:
-            conv = Conversation(
-                src_ip=src_ip, dst_ip=dst_ip, src_port=src_port, dst_port=dst_port,
-                protocol=transport, first_seen=ts, last_seen=ts,
-            )
-            try:
-                conv.stream_id = int(pkt.tcp.stream) if transport == "tcp" else None
-            except Exception:
-                conv.stream_id = None
-            conv_index[key] = conv
-        conv.packets += 1
-        conv.bytes += length
-        pnum = _num(pkt)
-        if pnum is not None and len(conv.packet_numbers) < 20:
-            conv.packet_numbers.append(pnum)
-        if ts is not None:
-            conv.timestamps.append(ts)
-            conv.last_seen = ts if conv.last_seen is None else max(conv.last_seen, ts)
-            conv.first_seen = ts if conv.first_seen is None else min(conv.first_seen, ts)
-        if src_ip == conv.src_ip:
-            conv.bytes_src_to_dst += length
-        else:
-            conv.bytes_dst_to_src += length
-
-    # TCP stream IDs support cross-packet authentication correlation.
-    stream_id = None
-    if transport == "tcp":
-        try:
-            stream_id = int(pkt.tcp.stream)
-        except Exception:
-            stream_id = None
-
-    # Protocol-specific extraction.
-    if "arp" in layers:
-        _handle_arp(ctx, pkt, ts)
-    if "dns" in layers:
-        _handle_dns(ctx, pkt, src_ip, dst_ip, ts)
-    if "http" in layers:
-        _handle_http(ctx, pkt, src_ip, dst_ip, dst_port, ts)
-    if "tls" in layers or "ssl" in layers:
-        _handle_tls(ctx, pkt, src_ip, dst_ip, dst_port, ts)
-    if "ftp" in layers:
-        _handle_ftp(ctx, pkt, src_ip, dst_ip, dst_port, ts)
-    if "telnet" in layers:
-        _handle_telnet(ctx, pkt, src_ip, dst_ip, dst_port, ts)
-    if "pop" in layers:
-        _handle_pop3(ctx, pkt, src_ip, dst_ip, dst_port, ts)
-    if "imap" in layers:
-        _handle_imap(ctx, pkt, src_ip, dst_ip, dst_port, ts)
-    if "smtp" in layers:
-        _handle_smtp(ctx, pkt, src_ip, dst_ip, dst_port, ts, stream_id, state)
-    if "ntlmssp" in layers:
-        _handle_ntlm(ctx, pkt, src_ip, dst_ip, ts, stream_id, state)
-    if "kerberos" in layers:
-        _handle_kerberos(ctx, pkt, src_ip, dst_ip, ts)
-
-    # Generic payload extraction finds artifacts that protocol fields may omit.
-    _handle_raw_payload(ctx, pkt, layers, src_ip, dst_ip, ts)
-
-
-def _handle_arp(ctx, pkt, ts):
-    try:
-        a = pkt.arp
-        ctx.arp_events.append(ArpEvent(
-            sender_ip=getattr(a, "src_proto_ipv4", None),
-            sender_mac=getattr(a, "src_hw_mac", None),
-            target_ip=getattr(a, "dst_proto_ipv4", None),
-            opcode=int(getattr(a, "opcode", 0) or 0),
-            timestamp=ts,
-            packet_number=_num(pkt),
-        ))
-    except Exception:
-        pass
-
-
-def _handle_dns(ctx, pkt, src_ip, dst_ip, ts):
-    try:
-        d = pkt.dns
-        # Ignore DNS records without a query name.
-        qname = getattr(d, "qry_name", None)
-        if not qname:
-            return
-        rcode = getattr(d, "flags_rcode", None)
-        rcode_map = {"0": "NOERROR", "3": "NXDOMAIN"}
-        response_ips = []
-        for attr in ("a", "aaaa"):
-            response_ips.extend(_all_field_values(d, attr))
-        ctx.dns_queries.append(DnsQuery(
-            query_name=qname,
-            qtype=_dns_qtype_name(getattr(d, "qry_type", None)),
-            response_ips=response_ips,
-            rcode=rcode_map.get(str(rcode), rcode),
-            timestamp=ts,
-            src_ip=src_ip,
-            dst_ip=dst_ip,
-            packet_number=_num(pkt),
-        ))
-    except Exception:
-        pass
-
-
-def _handle_http(ctx, pkt, src_ip, dst_ip, dst_port, ts):
-    try:
-        h = pkt.http
-        if hasattr(h, "request_method") or hasattr(h, "request_uri"):
-            req = HttpRequest(
-                method=getattr(h, "request_method", None),
-                host=getattr(h, "host", None),
-                uri=getattr(h, "request_uri", None),
-                user_agent=getattr(h, "user_agent", None),
-                authorization=getattr(h, "authorization", None),
-                cookies=getattr(h, "cookie", None),
-                body=getattr(h, "file_data", None),
-                src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port, timestamp=ts,
-                packet_number=_num(pkt),
-            )
-            ctx.http_requests.append(req)
-            # Preserve visible HTTP text for artifact detection.
-            for piece in (req.uri, req.cookies, req.authorization, req.body):
-                if piece:
-                    ctx.text_payloads.append({
-                        "text": str(piece), "protocol": "http",
-                        "src_ip": src_ip, "dst_ip": dst_ip,
-                        "packet_number": _num(pkt), "timestamp": ts,
-                    })
-        if hasattr(h, "response_code"):
-            ct = getattr(h, "content_type", None)
-            ctx.http_responses.append(HttpResponse(
-                status_code=_safe_int(getattr(h, "response_code", None)),
-                content_type=ct,
-                server=getattr(h, "server", None),
-                src_ip=src_ip, dst_ip=dst_ip, timestamp=ts, packet_number=_num(pkt),
-            ))
-            # Record only responses that look like file downloads.
-            disposition = getattr(h, "content_disposition", None)
-            filename = _filename_from_disposition(disposition)
-            if filename or _is_download_content_type(ct):
-                ext = os.path.splitext(filename or "")[1].lower() or None
-                ctx.file_transfers.append(FileTransfer(
-                    filename=filename, extension=ext, content_type=ct,
-                    protocol="http", src_ip=src_ip, dst_ip=dst_ip,
-                    direction="download", timestamp=ts, packet_number=_num(pkt),
-                ))
-    except Exception:
-        pass
-
-
-def _handle_tls(ctx, pkt, src_ip, dst_ip, dst_port, ts):
-    try:
-        layer = getattr(pkt, "tls", None) or getattr(pkt, "ssl", None)
-        if layer is None:
-            return
-
-        # TLS 1.3 reports its real version in the supported-versions extension.
-        version = (
-            getattr(layer, "handshake_extensions_supported_version", None)
-            or getattr(layer, "handshake_version", None)
-            or getattr(layer, "record_version", None)
-        )
-
-        subject_cn = _first(
-            getattr(layer, "x509sat_printablestring", None),
-            getattr(layer, "x509ce_dnsname", None),
-        )
-        issuer_cn = getattr(layer, "x509sat_printablestring_1", None)
-        not_after = (
-            getattr(layer, "x509af_utctime", None)
-            or getattr(layer, "x509af_notafter", None)
-            or getattr(layer, "x509af_utctime_1", None)
-        )
-
-        ctx.tls_sessions.append(TlsSession(
-            src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port or 443,
-            version=_tls_version_name(version),
-            cipher=getattr(layer, "handshake_ciphersuite", None),
-            sni=getattr(layer, "handshake_extensions_server_name", None),
-            cert_subject_cn=subject_cn,
-            cert_issuer_cn=issuer_cn,
-            cert_not_after=not_after,
-            cert_self_signed=_cert_is_self_signed(subject_cn, issuer_cn),
-            cert_expired=_cert_is_expired(not_after, ctx.end_time or ts),
-            timestamp=ts, packet_number=_num(pkt),
-        ))
-    except Exception:
-        pass
-
-
-def _handle_ftp(ctx, pkt, src_ip, dst_ip, dst_port, ts):
-    try:
-        f = pkt.ftp
-        cmd = getattr(f, "request_command", None)
-        arg = getattr(f, "request_arg", None)
-        if cmd and cmd.upper() in ("USER", "PASS"):
-            pa = _last_plaintext(ctx, "ftp", src_ip, dst_ip, dst_port)
-            if cmd.upper() == "USER":
-                pa.username = arg
-            else:
-                pa.password = arg
-            pa.timestamp = ts
-            pa.packet_number = _num(pkt)
-    except Exception:
-        pass
-
-
-def _handle_telnet(ctx, pkt, src_ip, dst_ip, dst_port, ts):
-    try:
-        data = getattr(pkt.telnet, "data", None)
-        if data:
-            ctx.text_payloads.append({
-                "text": str(data), "protocol": "telnet",
-                "src_ip": src_ip, "dst_ip": dst_ip,
-                "packet_number": _num(pkt), "timestamp": ts,
-            })
-    except Exception:
-        pass
-
-
-def _handle_pop3(ctx, pkt, src_ip, dst_ip, dst_port, ts):
-    """POP3 USER/PASS im Klartext."""
-    try:
-        p = pkt.pop
-        cmd = (getattr(p, "request_command", None) or "").upper()
-        param = getattr(p, "request_parameter", None)
-        if cmd in ("USER", "PASS") and param:
-            pa = _last_plaintext(ctx, "pop3", src_ip, dst_ip, dst_port)
-            if cmd == "USER":
-                pa.username = param
-            else:
-                pa.password = param
-            pa.timestamp = ts
-            pa.packet_number = _num(pkt)
-    except Exception:
-        pass
-
-
-def _handle_imap(ctx, pkt, src_ip, dst_ip, dst_port, ts):
-    """Extract clear-text IMAP LOGIN credentials."""
-    try:
-        i = pkt.imap
-        line = getattr(i, "request", None) or getattr(i, "request_command", None)
-        if not line:
-            return
-        parts = str(line).split()
-        # Accept both tagged and untagged IMAP LOGIN forms.
-        idx = next((k for k, tok in enumerate(parts) if tok.upper() == "LOGIN"), None)
-        if idx is not None and len(parts) >= idx + 3:
-            user = parts[idx + 1].strip('"')
-            pw = parts[idx + 2].strip('"')
-            pa = _last_plaintext(ctx, "imap", src_ip, dst_ip, dst_port)
-            pa.username, pa.password = user, pw
-            pa.timestamp = ts
-            pa.packet_number = _num(pkt)
-    except Exception:
-        pass
-
-
-def _handle_smtp(ctx, pkt, src_ip, dst_ip, dst_port, ts, stream_id, state):
-    """Extract Base64-encoded SMTP AUTH LOGIN or PLAIN credentials."""
-    try:
-        s = pkt.smtp
-        param = getattr(s, "req_parameter", None) or getattr(s, "request_parameter", None)
-        cmd = (getattr(s, "req_command", None) or getattr(s, "request_command", None) or "").upper()
-        if not param and not cmd:
-            return
-        pending = state["smtp_auth"].setdefault(stream_id, {"mech": None, "tokens": []})
-
-        text = f"{cmd} {param}".strip()
-        upper = text.upper()
-        if "AUTH LOGIN" in upper:
-            pending["mech"] = "login"
-            return
-        if "AUTH PLAIN" in upper:
-            pending["mech"] = "plain"
-            # AUTH PLAIN may carry its token on the command line.
-            tail = text.split()
-            if len(tail) >= 3:
-                creds = crackformats.decode_smtp_auth_plain(tail[2])
-                _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt)
-                state["smtp_auth"].pop(stream_id, None)
-            return
-
-        # AUTH LOGIN sends username and password in subsequent lines.
-        if pending["mech"] == "login" and param:
-            pending["tokens"].append(param)
-            if len(pending["tokens"]) >= 2:
-                user, pw = crackformats.decode_smtp_auth_login(
-                    pending["tokens"][0], pending["tokens"][1])
-                _emit_smtp(ctx, (user, pw), src_ip, dst_ip, dst_port, ts, pkt)
-                state["smtp_auth"].pop(stream_id, None)
-        elif pending["mech"] == "plain" and param:
-            creds = crackformats.decode_smtp_auth_plain(param)
-            _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt)
-            state["smtp_auth"].pop(stream_id, None)
-    except Exception:
-        pass
-
-
-def _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt):
-    if not creds:
-        return
-    user, pw = creds
-    if not (user or pw):
-        return
-    ctx.plaintext_auth.append(PlaintextAuth(
-        protocol="smtp", username=user, password=pw,
-        src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port,
-        timestamp=ts, packet_number=_num(pkt),
-    ))
-
-
-def _handle_ntlm(ctx, pkt, src_ip, dst_ip, ts, stream_id, state):
-    """Correlate NTLM challenge and response packets into hashcat material."""
-    try:
-        n = pkt.ntlmssp
-        msg_type = str(getattr(n, "messagetype", "") or "").lower()
-
-        # Retain the type-2 server challenge for the matching type-3 response.
-        server_chal = getattr(n, "ntlmserverchallenge", None)
-        if server_chal and ("0x00000002" in msg_type or msg_type.endswith("2")):
-            state["ntlm_chal"][stream_id] = str(server_chal)
-
-        # A type-3 message provides the account and NT response.
-        account = getattr(n, "auth_username", None)
-        domain = getattr(n, "auth_domain", None)
-        nt_resp = (getattr(n, "ntlmv2_response", None)
-                   or getattr(n, "auth_ntresponse", None))
-
-        if account or nt_resp:
-            chal = state["ntlm_chal"].get(stream_id)
-            am = AuthMaterial(
-                kind="ntlmssp", account=account, domain=domain,
-                src_ip=src_ip, dst_ip=dst_ip, protocol="ntlm",
-                timestamp=ts, packet_number=_num(pkt),
-                server_challenge=chal, nt_response=str(nt_resp) if nt_resp else None,
-            )
-            built = crackformats.build_netntlmv2(account, domain, chal, nt_resp)
-            if built:
-                am.kind = "netntlmv2"
-                am.hashcat = built["hashcat"]
-                am.hashcat_mode = built["hashcat_mode"]
-                am.hash_format = built["format"]
-            ctx.auth_material.append(am)
-    except Exception:
-        pass
-
-
-# Bound retained payload text for large captures.
-_MAX_TEXT_PAYLOADS = 20000
-
-
-def _handle_raw_payload(ctx, pkt, layers, src_ip, dst_ip, ts):
-    """Retain bounded TCP or UDP text for generic artifact detection."""
-    if len(ctx.text_payloads) >= _MAX_TEXT_PAYLOADS:
-        return
-    raw = None
-    try:
-        if "tcp" in layers and hasattr(pkt.tcp, "payload"):
-            raw = pkt.tcp.payload
-        elif "udp" in layers and hasattr(pkt.udp, "payload"):
-            raw = pkt.udp.payload
-        elif "data" in layers:
-            raw = getattr(pkt.data, "data", None)
-    except Exception:
-        raw = None
-    if not raw:
-        return
-
-    text = _bytes_from_hexish(str(raw))
-    if not text or not text.strip():
-        return
-    try:
-        proto = (pkt.highest_layer or "raw").lower()
-    except Exception:
-        proto = "raw"
-    ctx.text_payloads.append({
-        "text": text[:16384], "protocol": proto,
-        "src_ip": src_ip, "dst_ip": dst_ip,
-        "packet_number": _num(pkt), "timestamp": ts,
-    })
-
-
-def _bytes_from_hexish(raw: str) -> Optional[str]:
-    """Convert a hex-formatted pyshark payload to text when possible."""
-    s = str(raw).strip()
-    compact = s.replace(":", "").replace(".", "").replace(" ", "")
-    if compact and len(compact) % 2 == 0 and all(c in "0123456789abcdefABCDEF" for c in compact):
-        try:
-            return bytes.fromhex(compact).decode("utf-8", "ignore")
-        except Exception:
-            return None
-    # Some dissectors already expose decoded text.
-    return s
-
-
-def _handle_kerberos(ctx, pkt, src_ip, dst_ip, ts):
-    """Extract Kerberos AS-REP and TGS-REP material for offline analysis."""
-    try:
-        k = pkt.kerberos
-        msg_type = str(getattr(k, "msg_type", "") or "")
-        account = getattr(k, "cnamestring", None) or getattr(k, "cnamestring_1", None)
-        realm = getattr(k, "realm", None) or getattr(k, "crealm", None)
-        etype = getattr(k, "etype", None)
-        cipher = getattr(k, "cipher", None) or getattr(k, "enc_part_cipher", None)
-
-        am = AuthMaterial(
-            kind=f"kerberos_{msg_type}" if msg_type else "kerberos",
-            account=account, domain=realm,
-            src_ip=src_ip, dst_ip=dst_ip, protocol="kerberos",
-            timestamp=ts, packet_number=_num(pkt),
-            etype=str(etype) if etype is not None else None,
-        )
-        # Kerberos message types 11 and 13 are AS-REP and TGS-REP.
-        built = None
-        if msg_type == "11":
-            built = crackformats.build_krb5_asrep(account, realm, etype, cipher)
-        elif msg_type == "13":
-            spn = getattr(k, "snamestring", None)
-            built = crackformats.build_krb5_tgsrep(account, realm, spn, etype, cipher)
-        if built:
-            am.hashcat = built["hashcat"]
-            am.hashcat_mode = built["hashcat_mode"]
-            am.hash_format = built["format"]
-        ctx.auth_material.append(am)
-    except Exception:
-        pass
-
-
-# --------------------------------------------------------------------------- #
-# kleine Helfer
-# --------------------------------------------------------------------------- #
-
-def _last_plaintext(ctx, protocol, src_ip, dst_ip, dst_port) -> PlaintextAuth:
-    """Find or create the authentication record that combines USER and PASS."""
-    for pa in reversed(ctx.plaintext_auth):
-        if pa.protocol == protocol and pa.src_ip == src_ip and pa.dst_ip == dst_ip:
-            if pa.password is None or pa.username is None:
-                return pa
-    pa = PlaintextAuth(protocol=protocol, src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port)
-    ctx.plaintext_auth.append(pa)
-    return pa
-
-
-def _safe_int(v):
-    try:
-        return int(v)
-    except Exception:
-        return None
-
-
-def _tls_version_name(raw) -> Optional[str]:
-    mapping = {
-        "0x0300": "SSL 3.0", "0x0301": "TLS 1.0", "0x0302": "TLS 1.1",
-        "0x0303": "TLS 1.2", "0x0304": "TLS 1.3",
-        "768": "SSL 3.0", "769": "TLS 1.0", "770": "TLS 1.1",
-        "771": "TLS 1.2", "772": "TLS 1.3",
-    }
-    if raw is None:
-        return None
-    return mapping.get(str(raw).lower(), str(raw))
-
-
-def _first(*vals):
-    """Return the first non-empty dissector field value."""
-    for v in vals:
-        if v:
-            return v
-    return None
-
-
-def _all_field_values(layer, name) -> list[str]:
-    """Return every occurrence of a repeated pyshark field."""
-    out: list[str] = []
-    try:
-        field = getattr(layer, name, None)
-        if field is None:
-            return out
-        allf = getattr(field, "all_fields", None)
-        if allf:
-            for f in allf:
-                v = getattr(f, "show", None) or getattr(f, "showname_value", None)
-                if v:
-                    out.append(str(v))
-        else:
-            out.append(str(field))
-    except Exception:
-        pass
-    return out
-
-
-# tshark often exposes DNS query types as numeric values.
-_DNS_QTYPES = {
-    "1": "A", "2": "NS", "5": "CNAME", "6": "SOA", "12": "PTR", "15": "MX",
-    "16": "TXT", "28": "AAAA", "33": "SRV", "255": "ANY", "10": "NULL",
-    "43": "DS", "48": "DNSKEY", "257": "CAA", "65": "HTTPS", "64": "SVCB",
-}
-
-
-def _dns_qtype_name(raw) -> Optional[str]:
-    if raw is None:
-        return None
-    s = str(raw).strip()
-    return _DNS_QTYPES.get(s, s.upper())
-
-
-# Content types that indicate a file rather than ordinary web content.
-_DOWNLOAD_CT_MARKERS = (
-    "application/octet-stream", "application/x-msdownload", "application/x-dosexec",
-    "application/x-executable", "application/vnd.microsoft.portable-executable",
-    "application/zip", "application/x-rar", "application/x-7z", "application/x-tar",
-    "application/gzip", "application/x-msi", "application/x-sh",
-    "application/java-archive", "application/x-elf",
-)
-
-
-def _is_download_content_type(ct) -> bool:
-    if not ct:
-        return False
-    low = str(ct).lower()
-    return any(m in low for m in _DOWNLOAD_CT_MARKERS)
-
-
-def _filename_from_disposition(disposition) -> Optional[str]:
-    """Extract a filename from a Content-Disposition header."""
-    if not disposition:
-        return None
-    import re as _re
-    m = _re.search(r'filename\*?=(?:"([^"]+)"|([^;]+))', str(disposition), _re.IGNORECASE)
-    if not m:
-        return None
-    name = (m.group(1) or m.group(2) or "").strip().strip('"')
-    # Remove an optional RFC 5987 character-set prefix.
-    if "''" in name:
-        name = name.split("''", 1)[1]
-    return name or None
-
-
-def _cert_is_self_signed(subject_cn, issuer_cn) -> Optional[bool]:
-    """Return whether subject and issuer match, or None if data is incomplete."""
-    if not subject_cn or not issuer_cn:
-        return None
-    return str(subject_cn).strip().lower() == str(issuer_cn).strip().lower()
-
-
-def _cert_is_expired(not_after, now_epoch) -> Optional[bool]:
-    """Compare a parsed certificate expiry with the capture timestamp."""
-    if not not_after or now_epoch is None:
-        return None
-    exp = _parse_cert_time(not_after)
-    if exp is None:
-        return None
-    try:
-        return float(now_epoch) > exp
-    except (TypeError, ValueError):
-        return None
-
-
-def _parse_cert_time(raw) -> Optional[float]:
-    """Convert common X.509 time formats to a UTC epoch timestamp."""
-    import datetime as _dt
-
-    s = str(raw).strip()
-    # tshark may already provide an epoch value.
-    try:
-        return float(s)
-    except ValueError:
-        pass
-    fmts = (
-        "%Y-%m-%d %H:%M:%S",       # 2020-01-01 00:00:00
-        "%b %d %H:%M:%S %Y GMT",   # Jan  1 00:00:00 2020 GMT
-        "%y%m%d%H%M%SZ",           # UTCTime  200101000000Z
-        "%Y%m%d%H%M%SZ",           # GeneralizedTime
-        "%Y-%m-%dT%H:%M:%SZ",      # ISO 8601
-    )
-    for fmt in fmts:
-        try:
-            dt = _dt.datetime.strptime(s, fmt).replace(tzinfo=_dt.timezone.utc)
-            return dt.timestamp()
-        except ValueError:
-            continue
-    return None

django_project/analyzer/inventory.py

@@ -0,0 +1,438 @@
+"""Build a complete observation inventory from the capture context.
+
+Findings say what is wrong; the inventory says what was there.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Optional
+
+from .context import CaptureContext, Host, is_special
+
+_MAX_HOSTS = 200
+_MAX_ENDPOINTS = 400
+_MAX_DNS_NAMES = 500
+_MAX_TLS = 400
+
+# Well-known service names for ports without a dissected protocol.
+_PORT_NAMES = {
+    20: "ftp-data", 21: "ftp", 22: "ssh", 23: "telnet", 25: "smtp", 53: "dns", 67: "dhcp",
+    68: "dhcp", 69: "tftp", 80: "http", 110: "pop3", 123: "ntp", 137: "netbios-ns",
+    138: "netbios-dgm", 139: "netbios-ssn", 143: "imap", 161: "snmp", 162: "snmp-trap",
+    389: "ldap", 443: "https", 445: "smb", 465: "smtps", 500: "ike", 502: "modbus",
+    514: "syslog", 554: "rtsp", 587: "submission", 631: "ipp", 636: "ldaps", 853: "dot",
+    873: "rsync", 993: "imaps", 995: "pop3s", 1080: "socks", 1194: "openvpn",
+    1433: "mssql", 1521: "oracle", 1883: "mqtt", 1900: "ssdp", 2049: "nfs",
+    3306: "mysql", 3389: "rdp", 4500: "ipsec-nat-t", 5000: "upnp/http-alt", 5060: "sip",
+    5061: "sips", 5222: "xmpp", 5223: "apns", 5228: "gcm", 5353: "mdns", 5355: "llmnr",
+    5432: "postgresql", 5683: "coap", 5684: "coaps", 5900: "vnc", 6379: "redis",
+    6667: "irc", 8000: "http-alt", 8080: "http-alt", 8443: "https-alt", 8883: "mqtts",
+    8888: "http-alt", 9000: "http-alt", 9100: "jetdirect", 11211: "memcached",
+    27017: "mongodb", 49152: "upnp",
+}
+
+# Dissector helper names that never describe a service by themselves.
+_NOISE_PROTOS = {"x509ce", "x509sat", "x509af", "x509if", "pkix1implicit", "pkix1explicit",
+                 "pkcs-1", "ber", "tcp.segments", "data", "data-text-lines", "mime_multipart",
+                 "image-jfif", "image-gif", "png", "json", "xml", "urlencoded-form", "media"}
+
+
+def _clean(protos) -> list[str]:
+    return sorted(p for p in protos if p not in _NOISE_PROTOS)
+
+
+_CLEARTEXT_APP = {"http", "ftp", "telnet", "pop", "imap", "smtp", "snmp", "tftp", "mqtt",
+                  "coap", "rtsp", "sip", "ldap", "mbtcp", "modbus", "nbns", "smb", "smb2",
+                  "syslog", "irc", "xmpp", "vnc", "rdp"}
+
+
+def build_inventory(ctx: CaptureContext) -> dict:
+    inv: dict = {}
+    inv["capture"] = _capture_section(ctx)
+    inv["device"] = _device_section(ctx)
+    inv["hosts"] = _hosts_section(ctx)
+    inv["external_endpoints"] = _endpoints_section(ctx)
+    inv["protocols"] = _protocols_section(ctx)
+    inv["services"] = _services_section(ctx)
+    inv["tls"] = _tls_section(ctx)
+    inv["dns"] = _dns_section(ctx)
+    inv["http"] = _http_section(ctx)
+    inv["cleartext"] = _cleartext_section(ctx)
+    inv["coverage"] = _coverage_section(ctx)
+    return inv
+
+
+# --------------------------------------------------------------------------- #
+
+def _capture_section(ctx: CaptureContext) -> dict:
+    return {
+        "packets": ctx.packet_count,
+        "duration_seconds": round(ctx.duration, 3),
+        "start": ctx.start_time, "end": ctx.end_time,
+        "vlans": sorted(ctx.vlan_ids),
+        "hosts_total": len([h for h in ctx.hosts.values() if not is_special(h.ip)]),
+        "internal_hosts": len([h for h in ctx.hosts.values() if h.is_private and not is_special(h.ip)]),
+        "external_hosts": len([h for h in ctx.hosts.values() if not h.is_private]),
+        "conversations": len(ctx.conversations),
+        "extractor": ctx.extractor,
+    }
+
+
+def _host_dict(ctx: CaptureContext, h: Host) -> dict:
+    ext = {c.dst_ip for c in ctx.conversations if c.src_ip == h.ip and ctx.is_external(c.dst_ip)}
+    ext |= {c.src_ip for c in ctx.conversations if c.dst_ip == h.ip and ctx.is_external(c.src_ip)}
+    return {
+        "ip": h.ip,
+        "macs": sorted(h.macs),
+        "names": sorted(h.hostnames),
+        "dhcp_vendor_class": h.dhcp_vendor_class,
+        "is_device": ctx.is_device(h.ip),
+        "is_private": h.is_private,
+        "packets": h.packets,
+        "bytes_sent": h.bytes_sent, "bytes_received": h.bytes_received,
+        "first_seen": h.first_seen, "last_seen": h.last_seen,
+        "ports_offered": sorted(p for p in h.ports_offered if p is not None)[:60],
+        "ports_contacted": sorted(p for p in h.ports_contacted if p is not None)[:60],
+        "protocols": _clean(h.app_protocols)[:40],
+        "external_endpoints": len(ext),
+        "user_agents": sorted(h.user_agents)[:10],
+    }
+
+
+def _device_section(ctx: CaptureContext) -> Optional[dict]:
+    if not ctx.device_ip:
+        # Suggest likely devices so the operator can re-run with --device-ip.
+        cands = []
+        for h in ctx.hosts.values():
+            if not h.is_private or is_special(h.ip):
+                continue
+            score = h.packets
+            if h.hostnames:
+                score *= 2
+            cands.append((score, h))
+        cands.sort(key=lambda t: -t[0])
+        return {
+            "configured": False,
+            "candidates": [
+                {"ip": h.ip, "names": sorted(h.hostnames), "macs": sorted(h.macs), "packets": h.packets}
+                for _, h in cands[:5]
+            ],
+        }
+    h = ctx.hosts.get(ctx.device_ip)
+    if h is None:
+        return {"configured": True, "ip": ctx.device_ip, "seen": False,
+                "note": "The configured device address does not appear in the capture."}
+    d = _host_dict(ctx, h)
+    d.update({"configured": True, "seen": True})
+    # Everything the device talked to.
+    peers: dict[str, dict] = {}
+    for c in ctx.conversations:
+        if c.src_ip == ctx.device_ip:
+            peer, port = c.dst_ip, c.dst_port
+        elif c.dst_ip == ctx.device_ip:
+            peer, port = c.src_ip, c.src_port
+        else:
+            continue
+        e = peers.setdefault(peer, {"ip": peer, "external": ctx.is_external(peer), "ports": set(),
+                                    "flows": 0, "bytes_to_peer": 0, "bytes_from_peer": 0,
+                                    "protocols": set(), "names": set()})
+        e["flows"] += 1
+        if port:
+            e["ports"].add(port)
+        if c.src_ip == ctx.device_ip:
+            e["bytes_to_peer"] += c.bytes_src_to_dst
+            e["bytes_from_peer"] += c.bytes_dst_to_src
+        else:
+            e["bytes_to_peer"] += c.bytes_dst_to_src
+            e["bytes_from_peer"] += c.bytes_src_to_dst
+        e["protocols"] |= c.app_protocols
+    for peer, e in peers.items():
+        e["names"] = sorted(ctx.names_for_ip(peer))[:8]
+        e["ports"] = sorted(e["ports"])[:20]
+        e["protocols"] = _clean(e["protocols"])[:15]
+    d["peers"] = sorted(peers.values(), key=lambda e: -(e["bytes_to_peer"] + e["bytes_from_peer"]))[:100]
+    d["dns_queries"] = sorted({q.query_name.rstrip(".").lower() for q in ctx.dns_queries
+                               if q.src_ip == ctx.device_ip and not q.is_response and q.query_name})[:100]
+    d["tls_sessions"] = len([t for t in ctx.tls_sessions if t.src_ip == ctx.device_ip])
+    d["cleartext_flows"] = len([c for c in ctx.conversations
+                                if ctx.device_ip in (c.src_ip, c.dst_ip)
+                                and c.payload_bytes > 0 and not c.is_encrypted])
+    return d
+
+
+def _hosts_section(ctx: CaptureContext) -> list[dict]:
+    hosts = [h for h in ctx.hosts.values() if not is_special(h.ip) and h.is_private]
+    hosts.sort(key=lambda h: (-int(ctx.is_device(h.ip)), -h.packets))
+    return [_host_dict(ctx, h) for h in hosts[:_MAX_HOSTS]]
+
+
+def _endpoints_section(ctx: CaptureContext) -> list[dict]:
+    eps: dict[str, dict] = {}
+    for c in ctx.conversations:
+        if ctx.is_external(c.dst_ip) and not ctx.is_external(c.src_ip):
+            ext, internal, port = c.dst_ip, c.src_ip, c.dst_port
+            up, down = c.bytes_src_to_dst, c.bytes_dst_to_src
+        elif ctx.is_external(c.src_ip) and not ctx.is_external(c.dst_ip):
+            ext, internal, port = c.src_ip, c.dst_ip, c.src_port
+            up, down = c.bytes_dst_to_src, c.bytes_src_to_dst
+        else:
+            continue
+        e = eps.setdefault(ext, {"ip": ext, "ports": set(), "protocols": set(), "flows": 0,
+                                 "bytes_out": 0, "bytes_in": 0, "internal_hosts": set(),
+                                 "first_seen": c.first_seen, "last_seen": c.last_seen,
+                                 "device": False})
+        e["flows"] += 1
+        e["bytes_out"] += up
+        e["bytes_in"] += down
+        if port:
+            e["ports"].add(port)
+        e["protocols"] |= c.app_protocols
+        e["internal_hosts"].add(internal)
+        if ctx.is_device(internal):
+            e["device"] = True
+        if c.first_seen is not None:
+            e["first_seen"] = c.first_seen if e["first_seen"] is None else min(e["first_seen"], c.first_seen)
+        if c.last_seen is not None:
+            e["last_seen"] = c.last_seen if e["last_seen"] is None else max(e["last_seen"], c.last_seen)
+    out = []
+    for ip, e in eps.items():
+        e["names"] = sorted(ctx.names_for_ip(ip))[:8]
+        e["ports"] = sorted(e["ports"])[:20]
+        e["protocols"] = _clean(e["protocols"])[:15]
+        e["internal_hosts"] = sorted(e["internal_hosts"])[:10]
+        e["encrypted"] = bool(set(e["protocols"]) & {"tls", "quic", "ssh", "dtls"})
+        out.append(e)
+    out.sort(key=lambda e: (-int(e["device"]), -(e["bytes_out"] + e["bytes_in"])))
+    return out[:_MAX_ENDPOINTS]
+
+
+def _protocols_section(ctx: CaptureContext) -> dict:
+    skip = {"eth", "ethertype", "vlan", "frame", "sll", "llc", "tcp.segments", "data",
+            "data-text-lines", "x509ce", "x509sat", "x509af", "x509if", "pkix1implicit",
+            "pkix1explicit", "pkcs-1", "ber", "mime_multipart"}
+    app = {k: v for k, v in ctx.app_protocol_stats.items() if k not in skip}
+    return {
+        "by_protocol": dict(sorted(app.items(), key=lambda kv: -kv[1])),
+        "highest_layer": dict(sorted(ctx.protocol_stats.items(), key=lambda kv: -kv[1])[:30]),
+        "icmp": dict(ctx.icmp_stats),
+    }
+
+
+def _services_section(ctx: CaptureContext) -> list[dict]:
+    """Server-side services observed: who answered on which port with which protocol."""
+    svc: dict[tuple, dict] = {}
+    for c in ctx.conversations:
+        if not c.answered or not c.dst_port:
+            continue
+        key = (c.dst_ip, c.dst_port, c.protocol)
+        e = svc.setdefault(key, {"ip": c.dst_ip, "port": c.dst_port, "transport": c.protocol,
+                                 "protocols": set(), "clients": set(), "flows": 0,
+                                 "external": ctx.is_external(c.dst_ip), "device": ctx.is_device(c.dst_ip)})
+        e["flows"] += 1
+        e["clients"].add(c.src_ip)
+        e["protocols"] |= c.app_protocols
+    out = []
+    for e in svc.values():
+        protos = _clean(e["protocols"])
+        e["protocols"] = protos
+        e["service"] = _service_name(e["port"], protos)
+        e["encrypted"] = bool(set(protos) & {"tls", "quic", "ssh", "dtls"})
+        e["cleartext_app"] = bool(set(protos) & _CLEARTEXT_APP) and not e["encrypted"]
+        e["clients"] = sorted(e["clients"])[:10]
+        out.append(e)
+    out.sort(key=lambda e: (-int(e["device"]), int(e["external"]), e["ip"], e["port"]))
+    return out[:_MAX_ENDPOINTS]
+
+
+def _service_name(port: int, protos: list[str]) -> str:
+    for p in ("mqtt", "coap", "http", "rtsp", "ssh", "telnet", "ftp", "smtp", "pop", "imap",
+              "snmp", "dns", "mdns", "ssdp", "dhcp", "ntp", "tftp", "sip", "smb2", "smb", "ldap",
+              "mbtcp", "vnc", "rdp", "xmpp", "quic", "tls"):
+        if p in protos:
+            base = {"pop": "pop3", "mbtcp": "modbus"}.get(p, p)
+            if base == "tls":
+                return _PORT_NAMES.get(port, f"tls/{port}")
+            return base
+    return _PORT_NAMES.get(port, f"port {port}")
+
+
+def _tls_section(ctx: CaptureContext) -> dict:
+    sessions = []
+    for t in ctx.tls_sessions:
+        sessions.append({
+            "client": t.src_ip, "server": t.dst_ip, "port": t.dst_port, "transport": t.transport,
+            "sni": t.sni, "version": t.version, "cipher": t.cipher,
+            "client_max_version": t.client_version,
+            "client_offered_ciphers": len(t.client_offered_ciphers),
+            "alpn": t.alpn[:4], "handshake_complete": t.server_hello_seen,
+            "certificate": {
+                "subject": t.cert_subject_cn, "issuer": t.cert_issuer_cn, "san": t.cert_san[:5],
+                "not_after": t.cert_not_after, "self_signed": t.cert_self_signed,
+                "expired": t.cert_expired, "signature": t.cert_sig_algorithm,
+                "key": f"{t.cert_key_type} {t.cert_key_bits}" if t.cert_key_type else None,
+            } if t.cert_seen else None,
+            "mutual_tls": t.client_cert_seen,
+            "alerts": t.alerts,
+            "device": ctx.is_device(t.src_ip) or ctx.is_device(t.dst_ip),
+            "packet": t.packet_number,
+        })
+    sessions.sort(key=lambda s: (-int(s["device"]), s["client"] or "", s["server"] or ""))
+    versions: dict[str, int] = defaultdict(int)
+    ciphers: dict[str, int] = defaultdict(int)
+    for t in ctx.tls_sessions:
+        if t.version:
+            versions[t.version] += 1
+        if t.cipher:
+            ciphers[t.cipher] += 1
+    return {
+        "sessions_total": len(ctx.tls_sessions),
+        "versions": dict(versions), "ciphers": dict(ciphers),
+        "sessions": sessions[:_MAX_TLS],
+    }
+
+
+def _dns_section(ctx: CaptureContext) -> dict:
+    names: dict[str, dict] = {}
+    rcodes: dict[str, int] = defaultdict(int)
+    qtypes: dict[str, int] = defaultdict(int)
+    resolvers: dict[str, int] = defaultdict(int)
+    for q in ctx.dns_queries:
+        name = (q.query_name or "").rstrip(".").lower()
+        if not name:
+            continue
+        e = names.setdefault(name, {"name": name, "queries": 0, "clients": set(), "answers": set(),
+                                    "rcodes": set(), "types": set(), "transport": q.transport})
+        if q.is_response:
+            if q.rcode:
+                rcodes[q.rcode] += 1
+                e["rcodes"].add(q.rcode)
+            e["answers"].update(q.response_ips[:6])
+            if q.resolver:
+                resolvers[q.resolver] += 1
+        else:
+            e["queries"] += 1
+            if q.src_ip:
+                e["clients"].add(q.src_ip)
+        if q.qtype:
+            qtypes[q.qtype] += 1
+            e["types"].add(q.qtype)
+    out = []
+    for e in names.values():
+        e["clients"] = sorted(e["clients"])[:6]
+        e["answers"] = sorted(e["answers"])[:6]
+        e["rcodes"] = sorted(e["rcodes"])
+        e["types"] = sorted(e["types"])
+        e["device"] = ctx.device_ip in e["clients"] if ctx.device_ip else False
+        out.append(e)
+    out.sort(key=lambda e: (-int(e["device"]), -e["queries"], e["name"]))
+    return {
+        "unique_names": len(names),
+        "queries": sum(1 for q in ctx.dns_queries if not q.is_response),
+        "responses": sum(1 for q in ctx.dns_queries if q.is_response),
+        "rcodes": dict(rcodes), "query_types": dict(qtypes), "resolvers": dict(resolvers),
+        "names": out[:_MAX_DNS_NAMES],
+    }
+
+
+def _http_section(ctx: CaptureContext) -> dict:
+    hosts: dict[str, dict] = {}
+    for r in ctx.http_requests:
+        key = str(r.host or r.dst_ip or "?")
+        e = hosts.setdefault(key, {"host": key, "server_ip": r.dst_ip, "port": r.dst_port,
+                                   "requests": 0, "methods": set(), "uris": [], "clients": set(),
+                                   "user_agents": set(), "auth": False})
+        e["requests"] += 1
+        if r.method:
+            e["methods"].add(r.method)
+        if r.uri and len(e["uris"]) < 15 and r.uri not in e["uris"]:
+            e["uris"].append(r.uri)
+        if r.src_ip:
+            e["clients"].add(r.src_ip)
+        if r.user_agent:
+            e["user_agents"].add(r.user_agent[:100])
+        if r.authorization:
+            e["auth"] = True
+    out = []
+    for e in hosts.values():
+        e["methods"] = sorted(e["methods"])
+        e["clients"] = sorted(e["clients"])[:6]
+        e["user_agents"] = sorted(e["user_agents"])[:5]
+        e["device"] = bool(ctx.device_ip and (ctx.device_ip in e["clients"] or e["server_ip"] == ctx.device_ip))
+        out.append(e)
+    out.sort(key=lambda e: (-int(e["device"]), -e["requests"]))
+    status: dict[str, int] = defaultdict(int)
+    for resp in ctx.http_responses:
+        if resp.status_code is not None:
+            status[str(resp.status_code)] += 1
+    return {"requests": len(ctx.http_requests), "responses": len(ctx.http_responses),
+            "status_codes": dict(status), "hosts": out[:200],
+            "file_transfers": [
+                {"filename": f.filename, "extension": f.extension, "content_type": f.content_type,
+                 "size": f.size, "protocol": f.protocol, "direction": f.direction,
+                 "src_ip": f.src_ip, "dst_ip": f.dst_ip, "uri": f.uri, "packet": f.packet_number}
+                for f in ctx.file_transfers[:100]
+            ]}
+
+
+def _cleartext_section(ctx: CaptureContext) -> dict:
+    """Unencrypted application flows and clear-text services grouped by protocol."""
+    flows: dict[str, dict] = {}
+    for c in ctx.conversations:
+        if c.payload_bytes <= 0 or c.is_encrypted:
+            continue
+        protos = sorted(c.app_protocols & _CLEARTEXT_APP) or ([_PORT_NAMES[c.dst_port]] if c.dst_port in _PORT_NAMES and c.dst_port not in (53, 123, 67, 68, 5353, 1900, 137, 138) else [])
+        if not protos:
+            continue
+        for p in protos:
+            e = flows.setdefault(p, {"protocol": p, "flows": 0, "bytes": 0, "pairs": set(), "device": False})
+            e["flows"] += 1
+            e["bytes"] += c.payload_bytes
+            e["pairs"].add(f"{c.src_ip} -> {c.dst_ip}:{c.dst_port}")
+            if ctx.is_device(c.src_ip) or ctx.is_device(c.dst_ip):
+                e["device"] = True
+    out = []
+    for e in flows.values():
+        e["pairs"] = sorted(e["pairs"])[:20]
+        out.append(e)
+    out.sort(key=lambda e: (-int(e["device"]), -e["bytes"]))
+    services = [
+        {"protocol": s.protocol, "kind": s.kind, "value": s.value, "detail": s.detail,
+         "src_ip": s.src_ip, "dst_ip": s.dst_ip, "port": s.dst_port, "packet": s.packet_number}
+        for s in ctx.service_events[:200]
+    ]
+    return {"flows": out, "service_events": services,
+            "mqtt_events": len(ctx.mqtt_events),
+            "dhcp": [
+                {"type": d.message_type, "client_mac": d.client_mac, "client_ip": d.client_ip,
+                 "hostname": d.hostname, "vendor_class": d.vendor_class, "server": d.server_id}
+                for d in ctx.dhcp_events[:60]
+            ]}
+
+
+def _coverage_section(ctx: CaptureContext) -> dict:
+    notes = list(ctx.coverage_notes)
+    if ctx.truncated:
+        notes.insert(0, ctx.truncation_reason or "Extraction was truncated.")
+    encrypted_flows = sum(1 for c in ctx.conversations if c.is_encrypted)
+    payload_flows = sum(1 for c in ctx.conversations if c.payload_bytes > 0)
+    undecoded = [c for c in ctx.conversations if c.payload_bytes > 0 and not c.app_protocols]
+    undecoded_ports: dict[int, int] = defaultdict(int)
+    for c in undecoded:
+        if c.dst_port:
+            undecoded_ports[c.dst_port] += 1
+    if not ctx.device_ip:
+        notes.append("No device address was configured; findings are not attributed to a device under test.")
+    tls13 = sum(1 for t in ctx.tls_sessions if t.version == "TLS 1.3")
+    if tls13:
+        notes.append(f"{tls13} TLS 1.3 sessions: server certificates are encrypted and could not be inspected.")
+    return {
+        "complete": not ctx.truncated,
+        "notes": notes,
+        "flows_total": len(ctx.conversations),
+        "flows_with_payload": payload_flows,
+        "flows_encrypted": encrypted_flows,
+        "flows_undecoded": len(undecoded),
+        "undecoded_ports": dict(sorted(undecoded_ports.items(), key=lambda kv: -kv[1])[:15]),
+        "text_payloads_scanned": len(ctx.text_payloads),
+    }

django_project/analyzer/logutil.py

@@ -0,0 +1,78 @@
+"""Logging helpers: per-run log files and consistent formatting."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import threading
+from typing import Iterator, Optional
+
+LOG_FORMAT = "%(asctime)s %(levelname)-7s [%(name)s] %(message)s"
+DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+# Loggers that describe a capture analysis end to end.
+ANALYSIS_LOGGERS = ("analyzer", "IoTAnalyzer.analyze")
+
+
+class _ThreadFilter(logging.Filter):
+    """Only pass records emitted by the thread that opened the run log."""
+
+    def __init__(self, thread_id: int):
+        super().__init__()
+        self.thread_id = thread_id
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.thread == self.thread_id
+
+
+@contextlib.contextmanager
+def run_log(path: Optional[str], level: int = logging.DEBUG) -> Iterator[Optional[logging.Handler]]:
+    """Write everything this thread logs about the analysis to ``path``.
+
+    Concurrent analyses in other threads do not leak into the file.
+    """
+    if not path:
+        yield None
+        return
+    handler: Optional[logging.FileHandler] = None
+    try:
+        handler = logging.FileHandler(path, mode="w", encoding="utf-8")
+    except OSError as exc:
+        logging.getLogger("analyzer").warning("Run log %s could not be opened: %s", path, exc)
+        yield None
+        return
+    handler.setLevel(level)
+    handler.setFormatter(logging.Formatter(LOG_FORMAT, DATE_FORMAT))
+    handler.addFilter(_ThreadFilter(threading.get_ident()))
+    loggers = [logging.getLogger(n) for n in ANALYSIS_LOGGERS]
+    previous = [(lg, lg.level) for lg in loggers]
+    for lg in loggers:
+        lg.addHandler(handler)
+        if lg.level == logging.NOTSET or lg.level > level:
+            lg.setLevel(level)
+    try:
+        yield handler
+    finally:
+        for lg, lvl in previous:
+            lg.removeHandler(handler)
+            lg.setLevel(lvl)
+        handler.close()
+
+
+def configure_console(level: str = "INFO") -> None:
+    """Console logging for CLI use."""
+    logging.basicConfig(level=getattr(logging, level.upper(), logging.INFO),
+                        format=LOG_FORMAT, datefmt=DATE_FORMAT)
+
+
+def tail(path: str, lines: int = 400, max_bytes: int = 256 * 1024) -> str:
+    """Return the last ``lines`` lines of a text file."""
+    try:
+        with open(path, "rb") as fh:
+            fh.seek(0, 2)
+            size = fh.tell()
+            fh.seek(max(0, size - max_bytes))
+            data = fh.read().decode("utf-8", "replace")
+    except OSError:
+        return ""
+    return "\n".join(data.splitlines()[-lines:])

django_project/analyzer/reporting/_common.py

@@ -105,3 +105,182 @@ def evidence_line(f: Finding) -> str:
         kv = ", ".join(f"{k}={v}" for k, v in list(extras.items())[:6])
         parts.append(kv)
     return " | ".join(parts)
+
+
+# --------------------------------------------------------------------------- #
+# Inventory rendering shared by the Markdown and text renderers
+# --------------------------------------------------------------------------- #
+
+def _kb(n) -> str:
+    try:
+        n = int(n or 0)
+    except (TypeError, ValueError):
+        return "—"
+    if n >= 1024 * 1024:
+        return f"{n / (1024 * 1024):.1f} MB"
+    if n >= 1024:
+        return f"{n / 1024:.1f} kB"
+    return f"{n} B"
+
+
+def inventory_lines(inv: dict, *, md: bool = True) -> list[str]:
+    """Render the observation inventory as a list of lines.
+
+    ``md`` selects Markdown formatting; otherwise plain text is produced.
+    """
+    if not inv:
+        return []
+    h2 = (lambda t: f"## {t}") if md else (lambda t: t.upper())
+    h3 = (lambda t: f"### {t}") if md else (lambda t: f"  {t}")
+    b = (lambda t: f"- {t}") if md else (lambda t: f"  - {t}")
+    code = (lambda t: f"`{t}`") if md else (lambda t: str(t))
+    out: list[str] = []
+
+    cap = inv.get("capture") or {}
+    out.append(h2("Observation Inventory"))
+    out.append("")
+    out.append(b(f"Capture: {cap.get('packets', 0)} packets, {cap.get('duration_seconds', 0)} s, "
+                 f"{cap.get('internal_hosts', 0)} internal and {cap.get('external_hosts', 0)} external hosts, "
+                 f"{cap.get('conversations', 0)} flows"
+                 + (f", VLANs {', '.join(map(str, cap['vlans']))}" if cap.get("vlans") else "")
+                 + (f" · extractor {cap['extractor']}" if cap.get("extractor") else "")))
+
+    dev = inv.get("device") or {}
+    out.append("")
+    out.append(h3("Device under test"))
+    if not dev.get("configured"):
+        out.append(b("No device address configured."))
+        for c in dev.get("candidates", [])[:5]:
+            names = ", ".join(c.get("names") or []) or "—"
+            out.append(b(f"Candidate {code(c['ip'])} ({names}; {', '.join(c.get('macs') or []) or 'no MAC'}; {c.get('packets')} packets)"))
+    elif not dev.get("seen"):
+        out.append(b(f"{code(dev.get('ip'))}: {dev.get('note')}"))
+    else:
+        out.append(b(f"{code(dev['ip'])} · MAC {', '.join(dev.get('macs') or []) or '—'} · "
+                     f"names {', '.join(dev.get('names') or []) or '—'}"
+                     + (f" · DHCP vendor class {dev['dhcp_vendor_class']}" if dev.get("dhcp_vendor_class") else "")))
+        out.append(b(f"{dev.get('packets')} packets · sent {_kb(dev.get('bytes_sent'))} · received {_kb(dev.get('bytes_received'))} · "
+                     f"first seen {fmt_ts(dev.get('first_seen'))} · last seen {fmt_ts(dev.get('last_seen'))}"))
+        out.append(b(f"Services offered: {', '.join(map(str, dev.get('ports_offered') or [])) or 'none observed'} · "
+                     f"ports contacted: {', '.join(map(str, dev.get('ports_contacted') or [])) or '—'}"))
+        out.append(b(f"Protocols: {', '.join(dev.get('protocols') or []) or '—'} · TLS sessions: {dev.get('tls_sessions', 0)} · "
+                     f"clear-text flows: {dev.get('cleartext_flows', 0)}"))
+        if dev.get("dns_queries"):
+            out.append(b("DNS names queried: " + ", ".join(dev["dns_queries"][:30])))
+        peers = dev.get("peers") or []
+        if peers:
+            out.append(b(f"Peers ({len(peers)}):"))
+            for p in peers[:40]:
+                names = ", ".join(p.get("names") or [])
+                out.append(("    " if md else "      ") + b(
+                    f"{code(p['ip'])}{' (' + names + ')' if names else ''} · "
+                    f"{'external' if p.get('external') else 'internal'} · ports {', '.join(map(str, p.get('ports') or []))} · "
+                    f"{', '.join(p.get('protocols') or []) or 'undecoded'} · to peer {_kb(p.get('bytes_to_peer'))} / from peer {_kb(p.get('bytes_from_peer'))}"
+                ).lstrip())
+
+    hosts = inv.get("hosts") or []
+    if hosts:
+        out.append("")
+        out.append(h3(f"Internal hosts ({len(hosts)})"))
+        for hst in hosts[:40]:
+            names = ", ".join(hst.get("names") or [])
+            out.append(b(f"{code(hst['ip'])}{' (' + names + ')' if names else ''}"
+                         f"{' ← device' if hst.get('is_device') else ''} · MAC {', '.join(hst.get('macs') or []) or '—'} · "
+                         f"{hst.get('packets')} packets · offers {', '.join(map(str, hst.get('ports_offered') or [])) or '—'} · "
+                         f"{hst.get('external_endpoints', 0)} external endpoints"))
+
+    eps = inv.get("external_endpoints") or []
+    if eps:
+        out.append("")
+        out.append(h3(f"External endpoints ({len(eps)})"))
+        for e in eps[:60]:
+            names = ", ".join(e.get("names") or [])
+            out.append(b(f"{code(e['ip'])}{' (' + names + ')' if names else ''}{' ← device' if e.get('device') else ''} · "
+                         f"ports {', '.join(map(str, e.get('ports') or []))} · {', '.join(e.get('protocols') or []) or 'undecoded'} · "
+                         f"{'encrypted' if e.get('encrypted') else 'clear-text or unknown'} · out {_kb(e.get('bytes_out'))} / in {_kb(e.get('bytes_in'))}"))
+        if len(eps) > 60:
+            out.append(b(f"… and {len(eps) - 60} more (see JSON report)"))
+
+    svcs = inv.get("services") or []
+    if svcs:
+        out.append("")
+        out.append(h3(f"Services observed ({len(svcs)})"))
+        for s in svcs[:60]:
+            out.append(b(f"{code(str(s['ip']) + ':' + str(s['port']))} {s.get('service')} ({s.get('transport')})"
+                         f"{' ← device' if s.get('device') else ''} · {'encrypted' if s.get('encrypted') else 'clear-text' if s.get('cleartext_app') else 'unknown'} · "
+                         f"{s.get('flows')} flows from {len(s.get('clients') or [])} clients"))
+
+    protos = (inv.get("protocols") or {}).get("by_protocol") or {}
+    if protos:
+        out.append("")
+        out.append(h3("Protocols (packets)"))
+        out.append(b(", ".join(f"{k} {v}" for k, v in list(protos.items())[:30])))
+        icmp = (inv.get("protocols") or {}).get("icmp") or {}
+        if icmp:
+            out.append(b("ICMP: " + ", ".join(f"{k} {v}" for k, v in icmp.items())))
+
+    tls = inv.get("tls") or {}
+    if tls.get("sessions_total"):
+        out.append("")
+        out.append(h3(f"TLS posture ({tls['sessions_total']} handshakes)"))
+        out.append(b("Versions: " + ", ".join(f"{k} ×{v}" for k, v in (tls.get("versions") or {}).items())))
+        out.append(b("Cipher suites: " + ", ".join(f"{k} ×{v}" for k, v in (tls.get("ciphers") or {}).items())))
+        for s in (tls.get("sessions") or [])[:30]:
+            cert = s.get("certificate") or {}
+            cert_txt = ""
+            if cert:
+                cert_txt = f" · cert {cert.get('subject')} by {cert.get('issuer')}" + (f", expires {cert.get('not_after')}" if cert.get("not_after") else "")
+            out.append(b(f"{s.get('client')} → {code(str(s.get('server')) + ':' + str(s.get('port')))}"
+                         f"{' (' + s['sni'] + ')' if s.get('sni') else ''}{' ← device' if s.get('device') else ''} · "
+                         f"{s.get('version') or 'no ServerHello'} · {s.get('cipher') or '—'}"
+                         f"{' · client max ' + s['client_max_version'] if s.get('client_max_version') else ''}"
+                         f"{cert_txt}{' · mutual TLS' if s.get('mutual_tls') else ''}"
+                         f"{' · alerts ' + ', '.join(s['alerts']) if s.get('alerts') else ''}"))
+
+    dns = inv.get("dns") or {}
+    if dns.get("unique_names"):
+        out.append("")
+        out.append(h3(f"DNS ({dns.get('queries')} queries, {dns.get('responses')} responses, {dns['unique_names']} names)"))
+        out.append(b("Response codes: " + ", ".join(f"{k} {v}" for k, v in (dns.get("rcodes") or {}).items())
+                     + " · resolvers: " + ", ".join(f"{k} ({v})" for k, v in (dns.get("resolvers") or {}).items())))
+        for n in (dns.get("names") or [])[:60]:
+            out.append(b(f"{code(n['name'])}{' ← device' if n.get('device') else ''} · {n.get('queries')}× · "
+                         f"{', '.join(n.get('types') or [])} · {', '.join(n.get('answers') or []) or 'no answer seen'}"))
+
+    http = inv.get("http") or {}
+    if http.get("requests"):
+        out.append("")
+        out.append(h3(f"HTTP ({http['requests']} requests)"))
+        for hh in (http.get("hosts") or [])[:30]:
+            out.append(b(f"{code(hh['host'])}{' ← device' if hh.get('device') else ''} · {hh.get('requests')} requests · "
+                         f"{', '.join(hh.get('methods') or [])} · {', '.join(hh.get('uris') or [])[:200]}"
+                         f"{' · with Authorization header' if hh.get('auth') else ''}"))
+    for ft in (http.get("file_transfers") or [])[:20]:
+        out.append(b(f"File transfer: {ft.get('filename') or ft.get('uri') or ft.get('content_type')} "
+                     f"({ft.get('protocol')}, {ft.get('direction')}, {ft.get('src_ip')} → {ft.get('dst_ip')})"))
+
+    ct = inv.get("cleartext") or {}
+    if ct.get("flows") or ct.get("service_events") or ct.get("dhcp"):
+        out.append("")
+        out.append(h3("Clear-text protocols and service events"))
+        for f in ct.get("flows") or []:
+            out.append(b(f"{f['protocol']}{' ← device' if f.get('device') else ''}: {f.get('flows')} flows, {_kb(f.get('bytes'))} · "
+                         + "; ".join((f.get("pairs") or [])[:6])))
+        for s in (ct.get("service_events") or [])[:30]:
+            out.append(b(f"{s.get('protocol')} {s.get('kind')}: {s.get('value') or ''} {s.get('detail') or ''} "
+                         f"({s.get('src_ip')} → {s.get('dst_ip')})".strip()))
+        for d in (ct.get("dhcp") or [])[:20]:
+            out.append(b(f"DHCP {d.get('type')}: {d.get('client_mac')} {d.get('hostname') or ''} "
+                         f"{d.get('vendor_class') or ''} → {d.get('client_ip') or ''} (server {d.get('server') or '—'})".strip()))
+
+    cov = inv.get("coverage") or {}
+    out.append("")
+    out.append(h3("Coverage"))
+    out.append(b(f"{'Complete' if cov.get('complete') else 'PARTIAL'} extraction · {cov.get('flows_total', 0)} flows, "
+                 f"{cov.get('flows_encrypted', 0)} encrypted, {cov.get('flows_undecoded', 0)} with undecoded payload · "
+                 f"{cov.get('text_payloads_scanned', 0)} clear-text payloads scanned"))
+    if cov.get("undecoded_ports"):
+        out.append(b("Undecoded ports (flows): " + ", ".join(f"{k}: {v}" for k, v in cov["undecoded_ports"].items())))
+    for n in cov.get("notes") or []:
+        out.append(b(n))
+    return out

django_project/analyzer/reporting/markdown.py

@@ -6,6 +6,7 @@ from ..engine import AnalysisResult
 from ..findings import Severity
 from ._common import (
     SEVERITY_LABEL, fmt_ts, affected_line, evidence_line, artifact_line, sources_list,
+    inventory_lines,
 )
 
 
@@ -20,6 +21,12 @@ def render(result: AnalysisResult, extra: dict | None = None) -> str:
     out.append(f"- **Packets:** {m.get('packet_count', 0)}  ·  "
                f"**Duration:** {m.get('duration_seconds', 0)} s  ·  "
                f"**Findings:** {m.get('finding_count', 0)}")
+    if m.get("device_ip"):
+        out.append(f"- **Device under test:** `{m['device_ip']}`"
+                   + (f" (MAC {', '.join(m.get('device_macs') or [])})" if m.get("device_macs") else "")
+                   + f"  ·  device-related findings: {s.get('device_related', 0)}, other hosts: {s.get('other_hosts', 0)}")
+    if m.get("truncated"):
+        out.append(f"- **⚠️ Partial analysis:** {m.get('truncation_reason')}")
     out.append("")
 
     bysev = s.get("by_severity", {})
@@ -62,12 +69,18 @@ def render(result: AnalysisResult, extra: dict | None = None) -> str:
             for source in srcs:
                 out.append(f"    - {source}")
         out.append(f"- **Risk:** {f.risk_description}")
-        out.append(f"- **Affected:** {affected_line(f)}")
+        out.append(f"- **Affected:** {affected_line(f)}"
+                   + ("  ·  **involves device under test**" if f.evidence.get("device_related") else ""))
         out.append(f"- **Additional evidence:** {evidence_line(f)}")
         out.append(f"- **Recommendation:** {f.recommendation}")
         out.append(f"- **Finding ID:** `{f.finding_id}`")
         out.append("")
 
+    inv_lines = inventory_lines(result.inventory, md=True)
+    if inv_lines:
+        out.append("")
+        out.extend(inv_lines)
+
     # Optional URL and hash enrichment results.
     if extra:
         hashes = extra.get("hashes") or []

django_project/analyzer/reporting/text.py

@@ -6,6 +6,7 @@ from ..engine import AnalysisResult
 from ..findings import Severity
 from ._common import (
     SEVERITY_LABEL, affected_line, evidence_line, artifact_line, sources_list,
+    inventory_lines,
 )
 
 
@@ -21,6 +22,11 @@ def render(result: AnalysisResult, extra: dict | None = None) -> str:
     out.append(f"Capture : {m.get('capture_path', '-')}")
     out.append(f"Packets : {m.get('packet_count', 0)}   Duration: {m.get('duration_seconds', 0)}s")
     out.append(f"Findings: {m.get('finding_count', 0)}")
+    if m.get("device_ip"):
+        out.append(f"Device  : {m['device_ip']}  (device-related findings: {s.get('device_related', 0)}, "
+                   f"other hosts: {s.get('other_hosts', 0)})")
+    if m.get("truncated"):
+        out.append(f"WARNING : partial analysis - {m.get('truncation_reason')}")
     out.append("")
 
     bysev = s.get("by_severity", {})
@@ -62,6 +68,12 @@ def render(result: AnalysisResult, extra: dict | None = None) -> str:
         out.append(f"    Finding ID     : {f.finding_id}")
         out.append("")
 
+    inv_lines = inventory_lines(result.inventory, md=False)
+    if inv_lines:
+        out.append("-" * 70)
+        out.extend(inv_lines)
+        out.append("")
+
     if extra:
         hashes = extra.get("hashes") or []
         urls = extra.get("urls") or []

django_project/analyzer/scoring.py

@@ -34,14 +34,23 @@ def dedupe(findings: list[Finding]) -> list[Finding]:
 
 
 def sort_by_priority(findings: list[Finding]) -> list[Finding]:
-    """Sort by severity, confidence, and stable category order."""
+    """Sort by severity, device relevance, confidence, and stable category order."""
     return sorted(
         findings,
-        key=lambda f: (f.severity.rank, f.confidence.rank, -_category_order(f.category)),
+        key=lambda f: (f.severity.rank, _device_rank(f), f.confidence.rank,
+                       -_category_order(f.category)),
         reverse=True,
     )
 
 
+def _device_rank(f: Finding) -> int:
+    """Findings that involve the device under test outrank equally severe others."""
+    rel = f.evidence.get("device_related") if f.evidence else None
+    if rel is True:
+        return 1
+    return 0
+
+
 def summarize(findings: list[Finding]) -> dict:
     """Build the prioritized summary consumed by report renderers."""
     by_sev = {s.value: 0 for s in Severity}
@@ -56,10 +65,15 @@ def summarize(findings: list[Finding]) -> dict:
         and f.severity.rank >= Severity.HIGH.rank
     ]
 
+    device_related = sum(1 for f in findings if f.evidence.get("device_related") is True)
+    other_hosts = sum(1 for f in findings if f.evidence.get("device_related") is False)
+
     return {
         "total": len(findings),
         "by_severity": by_sev,
         "by_category": by_cat,
+        "device_related": device_related,
+        "other_hosts": other_hosts,
         "highlights": highlights,
         "top": sort_by_priority(findings)[:10],
     }

django_project/analyzer/tests/test_analyzer.py

@@ -48,7 +48,7 @@ class PlaintextCredentialTests(unittest.TestCase):
             authorization=f"Basic {token}", src_ip="10.0.0.5", dst_ip="93.184.216.34",
             dst_port=80))
         res = run(ctx)
-        f = next(f for f in res.findings if "HTTP Basic" in f.title)
+        f = next(f for f in res.findings if "Basic authentication over HTTP" in f.title)
         self.assertEqual(f.evidence["username"], "bob")
 
 
@@ -76,7 +76,8 @@ class InsecureProtocolTests(unittest.TestCase):
         ctx = CaptureContext()
         ctx.conversations.append(Conversation(
             src_ip="192.168.0.2", dst_ip="192.168.0.3", src_port=50000, dst_port=23,
-            protocol="tcp", packets=10, bytes=500))
+            protocol="tcp", packets=10, bytes=500, payload_bytes=200, synack_count=1,
+            app_protocols={"telnet"}))
         res = run(ctx)
         f = next(f for f in res.findings if f.category == Category.INSECURE_PROTOCOL)
         self.assertEqual(f.severity, Severity.CRITICAL)
@@ -88,7 +89,8 @@ class TlsWeaknessTests(unittest.TestCase):
         ctx = CaptureContext()
         ctx.tls_sessions.append(TlsSession(
             src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
-            version="TLS 1.0", cipher="TLS_RSA_WITH_RC4_128_SHA", sni="legacy.example"))
+            version="TLS 1.0", cipher="TLS_RSA_WITH_RC4_128_SHA", sni="legacy.example",
+            server_hello_seen=True))
         res = run(ctx)
         titles = [f.title for f in res.findings if f.category == Category.TLS_WEAKNESS]
         self.assertTrue(any("Obsolete TLS" in t for t in titles))
@@ -337,7 +339,7 @@ class TlsCertTests(unittest.TestCase):
             src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
             version="TLS 1.2", sni="iot.local",
             cert_subject_cn="iot.local", cert_issuer_cn="iot.local",
-            cert_self_signed=True))
+            cert_self_signed=True, cert_seen=True))
         res = run(ctx)
         self.assertTrue(any("self-signed" in f.title for f in res.findings))
 
@@ -345,20 +347,20 @@ class TlsCertTests(unittest.TestCase):
         ctx = CaptureContext()
         ctx.tls_sessions.append(TlsSession(
             src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
-            version="TLS 1.2", cert_expired=True))
+            version="TLS 1.2", cert_expired=True, cert_seen=True))
         res = run(ctx)
         self.assertTrue(any("expired" in f.title for f in res.findings))
 
 
 class CertHelperTests(unittest.TestCase):
     def test_self_signed_helper(self):
-        from analyzer.extract_pyshark import _cert_is_self_signed
+        from analyzer.extract_tshark import _cert_is_self_signed
         self.assertTrue(_cert_is_self_signed("device.local", "device.local"))
         self.assertFalse(_cert_is_self_signed("device.local", "DigiCert CA"))
         self.assertIsNone(_cert_is_self_signed("device.local", None))
 
     def test_expired_helper(self):
-        from analyzer.extract_pyshark import _cert_is_expired
+        from analyzer.extract_tshark import _cert_is_expired
         # The certificate expired before the 2025 capture timestamp.
         now = 1735689600.0  # 2025-01-01 UTC
         self.assertTrue(_cert_is_expired("2020-01-01 00:00:00", now))
@@ -366,7 +368,7 @@ class CertHelperTests(unittest.TestCase):
         self.assertIsNone(_cert_is_expired(None, now))
 
     def test_parse_cert_time_formats(self):
-        from analyzer.extract_pyshark import _parse_cert_time
+        from analyzer.extract_tshark import _parse_cert_time
         self.assertIsNotNone(_parse_cert_time("2020-01-01 00:00:00"))
         self.assertIsNotNone(_parse_cert_time("Jan  1 00:00:00 2020 GMT"))
         self.assertIsNotNone(_parse_cert_time("200101000000Z"))
@@ -490,7 +492,7 @@ class UserAgentTests(unittest.TestCase):
 
 class DnsQtypeTests(unittest.TestCase):
     def test_qtype_name_mapping(self):
-        from analyzer.extract_pyshark import _dns_qtype_name
+        from analyzer.extract_tshark import _dns_qtype_name
         self.assertEqual(_dns_qtype_name("16"), "TXT")
         self.assertEqual(_dns_qtype_name("1"), "A")
         self.assertEqual(_dns_qtype_name("txt"), "TXT")
@@ -516,14 +518,14 @@ class DnsQtypeTests(unittest.TestCase):
 
 class DownloadHelperTests(unittest.TestCase):
     def test_is_download_content_type(self):
-        from analyzer.extract_pyshark import _is_download_content_type
+        from analyzer.extract_tshark import _is_download_content_type
         self.assertTrue(_is_download_content_type("application/octet-stream"))
         self.assertTrue(_is_download_content_type("application/x-msdownload"))
         self.assertFalse(_is_download_content_type("text/html; charset=utf-8"))
         self.assertFalse(_is_download_content_type(None))
 
     def test_filename_from_disposition(self):
-        from analyzer.extract_pyshark import _filename_from_disposition
+        from analyzer.extract_tshark import _filename_from_disposition
         self.assertEqual(_filename_from_disposition('attachment; filename="update.exe"'), "update.exe")
         self.assertEqual(_filename_from_disposition("attachment; filename=payload.bin"), "payload.bin")
         self.assertIsNone(_filename_from_disposition("inline"))
@@ -537,7 +539,7 @@ class FileTransferDownloadTests(unittest.TestCase):
             filename=None, extension=None, content_type="application/x-msdownload",
             protocol="http", src_ip="1.2.3.4", dst_ip="10.0.0.5", direction="download"))
         res = run(ctx)
-        self.assertTrue(any("harmful file" in f.title for f in res.findings))
+        self.assertTrue(any("Executable" in f.title for f in res.findings))
 
     def test_disposition_filename_extension_flagged(self):
         from analyzer.context import FileTransfer
@@ -765,7 +767,7 @@ class DemoPayloadRegressionTests(unittest.TestCase):
         self.assertTrue(any("40.101.70.178" in u for u in urls))
 
     def test_hexish_payload_decoder(self):
-        from analyzer.extract_pyshark import _bytes_from_hexish
+        from analyzer.extract_tshark import _bytes_from_hexish
         # pyshark commonly exposes payload bytes as colon-separated hex.
         self.assertEqual(_bytes_from_hexish("47:45:54"), "GET")
         self.assertEqual(_bytes_from_hexish("already text"), "already text")

django_project/analyzer/tests/test_regressions.py

@@ -0,0 +1,382 @@
+"""Regression tests for detector precision, device attribution and the inventory.
+
+Every case here mirrors a false positive or a gap that was observed on real
+captures. Synthetic contexts keep the suite independent of tshark; the final
+class exercises the real extractor when tshark and a capture are available.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import unittest
+
+from analyzer import AnalysisEngine, render, FORMATS
+from analyzer.context import (
+    CaptureContext, Conversation, DnsQuery, HttpRequest, HttpResponse, TlsSession,
+    ArpEvent, PlaintextAuth, AuthAttempt, FileTransfer, DhcpEvent, ServiceEvent,
+)
+from analyzer.findings import Category, Severity
+from analyzer import tls_tables
+
+
+def run(ctx):
+    return AnalysisEngine().analyze_context(ctx)
+
+
+def titles(res):
+    return [f.title for f in res.findings]
+
+
+class FalsePositiveRegressionTests(unittest.TestCase):
+    def test_bulk_flow_is_not_beaconing(self):
+        ctx = CaptureContext()
+        conv = Conversation(src_ip="192.168.1.50", dst_ip="8.8.4.4", src_port=40000,
+                            dst_port=443, protocol="tcp", stream_id=3, packets=200,
+                            bytes=200 * 1400)
+        conv.timestamps = [1000 + i * 0.01 for i in range(200)]  # Line-rate pacing
+        ctx.conversations.append(conv)
+        res = run(ctx)
+        self.assertNotIn(Category.MALWARE_BEACONING, {f.category for f in res.findings})
+
+    def test_beaconing_to_known_provider_is_low(self):
+        ctx = CaptureContext()
+        ctx.dns_queries.append(DnsQuery(query_name="courier.push.apple.com", is_response=True,
+                                        response_ips=["17.57.146.22"], rcode="NOERROR"))
+        for i in range(8):
+            t = float(i * 60)
+            ctx.conversations.append(Conversation(
+                src_ip="192.168.1.77", dst_ip="17.57.146.22", src_port=40000 + i, dst_port=5223,
+                protocol="tcp", packets=2, bytes=200, first_seen=t, last_seen=t + 1,
+                stream_id=i, timestamps=[t, t + 1]))
+        res = run(ctx)
+        beacons = [f for f in res.findings if f.category == Category.MALWARE_BEACONING]
+        self.assertTrue(beacons)
+        self.assertEqual(beacons[0].severity, Severity.LOW)
+
+    def test_beaconing_to_unnamed_ip_is_high(self):
+        ctx = CaptureContext()
+        for i in range(8):
+            t = float(i * 60)
+            ctx.conversations.append(Conversation(
+                src_ip="192.168.1.77", dst_ip="45.33.32.156", src_port=40000 + i, dst_port=4444,
+                protocol="tcp", packets=2, bytes=200, first_seen=t, last_seen=t + 1,
+                stream_id=i, timestamps=[t, t + 1]))
+        res = run(ctx)
+        beacons = [f for f in res.findings if f.category == Category.MALWARE_BEACONING]
+        self.assertTrue(beacons)
+        self.assertEqual(beacons[0].severity, Severity.HIGH)
+
+    def test_dhcp_source_address_is_not_arp_spoofing(self):
+        ctx = CaptureContext()
+        h = ctx.host("0.0.0.0")
+        h.macs = {"aa:aa:aa:aa:aa:01", "aa:aa:aa:aa:aa:02"}
+        res = run(ctx)
+        self.assertNotIn(Category.MITM, {f.category for f in res.findings})
+
+    def test_javascript_asset_is_not_malware(self):
+        ctx = CaptureContext()
+        ctx.http_requests.append(HttpRequest(
+            method="GET", host="192.168.1.50", uri="/static/app.js", src_ip="192.168.1.10",
+            dst_ip="192.168.1.50", dst_port=80, packet_number=1))
+        res = run(ctx)
+        self.assertFalse(any("Executable" in t for t in titles(res)))
+
+    def test_normal_arp_replies_are_not_a_flood(self):
+        ctx = CaptureContext()
+        ctx.start_time, ctx.end_time = 0.0, 3600.0
+        for i in range(60):
+            ctx.arp_events.append(ArpEvent(sender_ip=f"192.168.1.{i % 20 + 1}",
+                                           sender_mac=f"bb:bb:bb:bb:bb:{i % 20:02x}",
+                                           opcode=2, timestamp=i * 60.0, packet_number=100 + i))
+        res = run(ctx)
+        self.assertFalse(any("ARP reply" in t for t in titles(res)))
+
+    def test_arp_flood_by_rate_is_flagged(self):
+        ctx = CaptureContext()
+        ctx.start_time, ctx.end_time = 0.0, 60.0
+        for i in range(400):
+            ctx.arp_events.append(ArpEvent(sender_ip="192.168.1.1", sender_mac="cc:cc:cc:cc:cc:cc",
+                                           opcode=2, timestamp=i * 0.1, packet_number=i + 1))
+        res = run(ctx)
+        self.assertTrue(any("ARP reply" in t for t in titles(res)))
+
+    def test_nxdomain_ratio_counts_responses_only(self):
+        ctx = CaptureContext()
+        for i in range(30):
+            ctx.dns_queries.append(DnsQuery(query_name=f"host{i}.example.net", is_response=False,
+                                            src_ip="10.0.0.9", dst_ip="10.0.0.1"))
+            ctx.dns_queries.append(DnsQuery(query_name=f"host{i}.example.net", is_response=True,
+                                            rcode="NXDOMAIN" if i < 15 else "NOERROR",
+                                            src_ip="10.0.0.1", dst_ip="10.0.0.9"))
+        res = run(ctx)
+        self.assertTrue(any("NXDOMAIN" in t for t in titles(res)))
+
+    def test_http_403_is_not_bruteforce(self):
+        ctx = CaptureContext()
+        for i in range(12):
+            ctx.http_responses.append(HttpResponse(status_code=403, src_ip="10.0.0.1",
+                                                   dst_ip="10.0.0.9", packet_number=i + 1))
+        res = run(ctx)
+        self.assertFalse(any("failed HTTP" in t for t in titles(res)))
+
+    def test_cdn_with_many_named_subdomains_is_low(self):
+        ctx = CaptureContext()
+        for i in range(45):
+            ctx.dns_queries.append(DnsQuery(query_name=f"edge-{i}.cdn-vendor.example",
+                                            src_ip="10.0.0.9", dst_ip="10.0.0.1"))
+        res = run(ctx)
+        f = next(f for f in res.findings if "subdomains" in f.title)
+        self.assertEqual(f.severity, Severity.LOW)
+
+
+class TlsPrecisionTests(unittest.TestCase):
+    def test_cipher_name_mapping_from_tshark_values(self):
+        self.assertEqual(tls_tables.cipher_name("0xc02f"), "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256")
+        self.assertEqual(tls_tables.cipher_name("4866"), "TLS_AES_256_GCM_SHA384")
+        self.assertEqual(tls_tables.version_name("0x0304"), "TLS 1.3")
+        self.assertTrue(tls_tables.is_grease(0x9a9a))
+        self.assertFalse(tls_tables.is_grease(0xc02f))
+
+    def test_negotiated_weak_cipher_is_high(self):
+        ctx = CaptureContext()
+        ctx.tls_sessions.append(TlsSession(src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
+                                           version="TLS 1.2", cipher="TLS_RSA_WITH_RC4_128_MD5",
+                                           server_hello_seen=True, sni="x.example"))
+        res = run(ctx)
+        f = next(f for f in res.findings if "Weak TLS cipher" in f.title)
+        self.assertEqual(f.severity, Severity.HIGH)
+
+    def test_client_offered_weak_cipher_is_medium_for_device(self):
+        ctx = CaptureContext(device_ip="10.0.0.1")
+        ctx.tls_sessions.append(TlsSession(src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=8883,
+                                           client_hello_seen=True, client_version="TLS 1.2",
+                                           client_offered_ciphers=["TLS_RSA_WITH_3DES_EDE_CBC_SHA",
+                                                                   "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"]))
+        res = run(ctx)
+        f = next(f for f in res.findings if "offers broken cipher" in f.title)
+        self.assertEqual(f.severity, Severity.MEDIUM)
+        self.assertEqual(f.affected_device, "10.0.0.1")
+
+    def test_application_data_only_session_is_not_judged(self):
+        ctx = CaptureContext()
+        ctx.tls_sessions.append(TlsSession(src_ip="10.0.0.1", dst_ip="1.2.3.4", version="TLS 1.0"))
+        res = run(ctx)
+        self.assertFalse(any("Obsolete" in t for t in titles(res)))
+
+    def test_sha1_certificate_flagged(self):
+        ctx = CaptureContext()
+        ctx.tls_sessions.append(TlsSession(src_ip="10.0.0.1", dst_ip="1.2.3.4", version="TLS 1.2",
+                                           cert_seen=True, cert_sig_algorithm="sha1",
+                                           cert_key_type="RSA", cert_key_bits=1024))
+        res = run(ctx)
+        f = next(f for f in res.findings if "certificate" in f.title.lower())
+        self.assertIn("SHA1", f.title)
+        self.assertIn("1024", f.title)
+
+
+class NewSignalTests(unittest.TestCase):
+    def test_json_login_over_http(self):
+        ctx = CaptureContext()
+        ctx.http_requests.append(HttpRequest(
+            method="POST", host="192.168.1.50", uri="/api/login", content_type="application/json",
+            body='{"username":"admin","password":"12345678"}', src_ip="192.168.1.10",
+            dst_ip="192.168.1.50", dst_port=80, packet_number=4))
+        res = run(ctx)
+        f = next(f for f in res.findings if "Login form" in f.title)
+        self.assertEqual(f.evidence["username"], "admin")
+        self.assertNotEqual(f.evidence["password"], "12345678")
+
+    def test_query_string_credentials(self):
+        ctx = CaptureContext()
+        ctx.http_requests.append(HttpRequest(
+            method="GET", host="cam.local", uri="/cgi-bin/login?user=root&pwd=toor",
+            src_ip="192.168.1.10", dst_ip="192.168.1.50", dst_port=80))
+        res = run(ctx)
+        self.assertTrue(any("Login form" in t for t in titles(res)))
+
+    def test_mqtt_credentials(self):
+        ctx = CaptureContext(device_ip="192.168.1.60")
+        ctx.plaintext_auth.append(PlaintextAuth(protocol="mqtt", username="dev01", password="pw",
+                                                src_ip="192.168.1.60", dst_ip="10.1.1.1", dst_port=1883))
+        res = run(ctx)
+        f = next(f for f in res.findings if "MQTT" in f.title)
+        self.assertEqual(f.severity, Severity.CRITICAL)
+        self.assertEqual(f.affected_device, "192.168.1.60")
+
+    def test_snmp_community(self):
+        ctx = CaptureContext()
+        ctx.plaintext_auth.append(PlaintextAuth(protocol="snmp", password="public",
+                                                src_ip="10.0.0.2", dst_ip="10.0.0.3", dst_port=161))
+        res = run(ctx)
+        self.assertTrue(any("SNMP community" in t for t in titles(res)))
+
+    def test_failed_logins_from_auth_attempts(self):
+        ctx = CaptureContext()
+        for i in range(6):
+            ctx.auth_attempts.append(AuthAttempt(protocol="ftp", success=False, src_ip="10.0.0.9",
+                                                 dst_ip="10.0.0.5", dst_port=21, username="admin",
+                                                 packet_number=i + 1))
+        res = run(ctx)
+        self.assertTrue(any("failed FTP" in t for t in titles(res)))
+
+    def test_firmware_over_http_to_device(self):
+        ctx = CaptureContext(device_ip="192.168.1.60")
+        ctx.file_transfers.append(FileTransfer(filename="fw_v2.bin", extension=".bin",
+                                               content_type="application/octet-stream", protocol="http",
+                                               src_ip="1.2.3.4", dst_ip="192.168.1.60", direction="download"))
+        res = run(ctx)
+        f = next(f for f in res.findings if "Firmware" in f.title)
+        self.assertEqual(f.severity, Severity.HIGH)
+        self.assertEqual(f.affected_device, "192.168.1.60")
+
+    def test_dissected_mqtt_flow_is_flagged_by_protocol_not_port(self):
+        ctx = CaptureContext()
+        ctx.conversations.append(Conversation(src_ip="192.168.1.60", dst_ip="10.1.1.1", src_port=5000,
+                                              dst_port=4711, protocol="tcp", packets=20, bytes=2000,
+                                              payload_bytes=800, synack_count=1, app_protocols={"mqtt"}))
+        res = run(ctx)
+        self.assertTrue(any("MQTT" in t and "4711" in t for t in titles(res)))
+
+    def test_encrypted_flow_on_port_80_not_flagged(self):
+        ctx = CaptureContext()
+        ctx.conversations.append(Conversation(src_ip="192.168.1.60", dst_ip="10.1.1.1", src_port=5000,
+                                              dst_port=80, protocol="tcp", packets=20, bytes=2000,
+                                              payload_bytes=800, synack_count=1, app_protocols={"tls"}))
+        res = run(ctx)
+        self.assertNotIn(Category.INSECURE_PROTOCOL, {f.category for f in res.findings})
+
+    def test_unanswered_syns_are_a_scan(self):
+        ctx = CaptureContext()
+        for port in range(20, 30):
+            ctx.conversations.append(Conversation(src_ip="10.0.0.66", dst_ip="10.0.0.5", src_port=50000 + port,
+                                                  dst_port=port, protocol="tcp", packets=1, bytes=60,
+                                                  syn_count=1))
+        res = run(ctx)
+        f = next(f for f in res.findings if "Port scan" in f.title)
+        self.assertEqual(f.evidence["unanswered_ports"], 10)
+
+    def test_multiple_dhcp_servers(self):
+        ctx = CaptureContext()
+        ctx.dhcp_events.append(DhcpEvent(message_type="offer", server_id="192.168.1.1", src_ip="192.168.1.1"))
+        ctx.dhcp_events.append(DhcpEvent(message_type="offer", server_id="192.168.1.200", src_ip="192.168.1.200"))
+        res = run(ctx)
+        self.assertTrue(any("DHCP servers" in t for t in titles(res)))
+
+    def test_public_resolver_bypass(self):
+        ctx = CaptureContext(device_ip="192.168.1.60")
+        ctx.dns_queries.append(DnsQuery(query_name="api.vendor.example", src_ip="192.168.1.60", dst_ip="8.8.8.8"))
+        res = run(ctx)
+        self.assertTrue(any("external resolver" in t for t in titles(res)))
+
+    def test_device_services_listed(self):
+        ctx = CaptureContext(device_ip="192.168.1.60")
+        ctx.conversations.append(Conversation(src_ip="192.168.1.10", dst_ip="192.168.1.60", src_port=5000,
+                                              dst_port=8080, protocol="tcp", packets=5, bytes=500,
+                                              synack_count=1, app_protocols={"http"}, payload_bytes=100))
+        res = run(ctx)
+        self.assertTrue(any("offers a network service on port 8080" in t for t in titles(res)))
+
+
+class DeviceAttributionTests(unittest.TestCase):
+    def test_device_findings_sorted_first_within_severity(self):
+        ctx = CaptureContext(device_ip="192.168.1.60")
+        ctx.plaintext_auth.append(PlaintextAuth(protocol="ftp", username="a", password="b",
+                                                src_ip="192.168.1.10", dst_ip="10.0.0.5", dst_port=21))
+        ctx.plaintext_auth.append(PlaintextAuth(protocol="ftp", username="c", password="d",
+                                                src_ip="192.168.1.60", dst_ip="10.0.0.5", dst_port=21))
+        res = run(ctx)
+        self.assertEqual(res.findings[0].affected_device, "192.168.1.60")
+        self.assertTrue(res.findings[0].evidence["device_related"])
+        self.assertFalse(res.findings[1].evidence["device_related"])
+        self.assertEqual(res.summary["device_related"], 1)
+        self.assertEqual(res.summary["other_hosts"], 1)
+
+    def test_no_device_leaves_attribution_unset(self):
+        ctx = CaptureContext()
+        ctx.plaintext_auth.append(PlaintextAuth(protocol="ftp", username="a", password="b",
+                                                src_ip="192.168.1.10", dst_ip="10.0.0.5", dst_port=21))
+        res = run(ctx)
+        self.assertNotIn("device_related", res.findings[0].evidence)
+        self.assertFalse(res.inventory["device"]["configured"])
+
+
+class InventoryTests(unittest.TestCase):
+    def _ctx(self):
+        ctx = CaptureContext(path="x.pcap", device_ip="192.168.1.60")
+        ctx.start_time, ctx.end_time = 0.0, 120.0
+        h = ctx.host("192.168.1.60")
+        h.macs.add("00:11:22:33:44:55")
+        h.hostnames.add("sensor-1")
+        ctx.host("45.33.32.157")
+        ctx.conversations.append(Conversation(src_ip="192.168.1.60", dst_ip="45.33.32.157", src_port=5000,
+                                              dst_port=8883, protocol="tcp", packets=50, bytes=5000,
+                                              bytes_src_to_dst=3000, bytes_dst_to_src=2000, payload_bytes=4000,
+                                              synack_count=1, app_protocols={"tls"}, first_seen=1.0, last_seen=100.0))
+        ctx.dns_queries.append(DnsQuery(query_name="mqtt.vendor.example", src_ip="192.168.1.60", dst_ip="192.168.1.1"))
+        ctx.dns_queries.append(DnsQuery(query_name="mqtt.vendor.example", is_response=True, rcode="NOERROR",
+                                        response_ips=["45.33.32.157"], src_ip="192.168.1.1", dst_ip="192.168.1.60"))
+        ctx.tls_sessions.append(TlsSession(src_ip="192.168.1.60", dst_ip="45.33.32.157", dst_port=8883,
+                                           version="TLS 1.2", cipher="TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                                           sni="mqtt.vendor.example", server_hello_seen=True, client_hello_seen=True,
+                                           client_version="TLS 1.2", cert_seen=True, cert_subject_cn="*.vendor.example",
+                                           cert_issuer_cn="Example CA", client_cert_seen=True))
+        return ctx
+
+    def test_inventory_sections_and_device_focus(self):
+        res = run(self._ctx())
+        inv = res.inventory
+        for key in ("capture", "device", "hosts", "external_endpoints", "protocols", "services",
+                    "tls", "dns", "http", "cleartext", "coverage"):
+            self.assertIn(key, inv)
+        self.assertTrue(inv["device"]["configured"])
+        self.assertEqual(inv["device"]["names"], ["sensor-1"])
+        self.assertEqual(inv["device"]["peers"][0]["names"], ["mqtt.vendor.example"])
+        self.assertTrue(inv["external_endpoints"][0]["device"])
+        self.assertEqual(inv["external_endpoints"][0]["names"], ["mqtt.vendor.example"])
+        self.assertTrue(inv["tls"]["sessions"][0]["mutual_tls"])
+        self.assertEqual(inv["services"][0]["service"], "mqtts")
+        self.assertTrue(inv["coverage"]["complete"])
+
+    def test_inventory_survives_roundtrip_and_renders(self):
+        import json
+        from analyzer import result_from_dict
+        res = run(self._ctx())
+        rebuilt = result_from_dict(json.loads(json.dumps(res.to_dict())))
+        self.assertEqual(rebuilt.inventory["device"]["ip"], "192.168.1.60")
+        for fmt in FORMATS:
+            out = render(rebuilt, fmt)
+            self.assertIn("192.168.1.60", out)
+            self.assertIn("mqtt.vendor.example", out)
+
+
+@unittest.skipUnless(shutil.which("tshark"), "tshark is not installed")
+class RealCaptureTests(unittest.TestCase):
+    """Golden checks against the bundled demo capture through the real extractor."""
+
+    def _capture(self, name):
+        here = os.path.dirname(os.path.abspath(__file__))
+        for base in (os.path.join(here, "..", "..", "..", "demo_files"),
+                     os.environ.get("ANALYZER_DEMO_DIR", "")):
+            p = os.path.join(base, name) if base else None
+            if p and os.path.exists(p):
+                return p
+        return None
+
+    def test_demo_capture_end_to_end(self):
+        path = self._capture("demo_capture.pcapng")
+        if not path:
+            self.skipTest("demo capture not available")
+        from analyzer.extract_tshark import build_context_from_pcap
+        from analyzer.artifacts import extract_urls, extract_hashes
+        ctx = build_context_from_pcap(path)
+        self.assertEqual(ctx.packet_count, 2)
+        self.assertFalse(ctx.truncated)
+        self.assertEqual(len(ctx.http_requests), 1)
+        self.assertTrue(ctx.http_requests[0].body.startswith("5f4dcc3b"))  # Decoded, not hex
+        urls = {u.value for u in extract_urls(ctx)}
+        self.assertIn("eu.iot.dreame.tech", urls)  # From the retransmitted, undissected segment
+        self.assertGreaterEqual(len(extract_hashes(ctx)), 10)
+        res = AnalysisEngine().analyze_context(ctx)
+        self.assertTrue(any("HTTP" in f.title for f in res.findings))
+        self.assertIn("http", res.inventory["cleartext"]["flows"][0]["protocol"])

django_project/analyzer/tls_tables.py

@@ -0,0 +1,503 @@
+"""TLS cipher-suite and version name tables, generated from ``tshark -G values``."""
+
+from __future__ import annotations
+
+CIPHER_SUITES: dict[int, str] = {
+    0: 'TLS_NULL_WITH_NULL_NULL',
+    1: 'TLS_RSA_WITH_NULL_MD5',
+    2: 'TLS_RSA_WITH_NULL_SHA',
+    3: 'TLS_RSA_EXPORT_WITH_RC4_40_MD5',
+    4: 'TLS_RSA_WITH_RC4_128_MD5',
+    5: 'TLS_RSA_WITH_RC4_128_SHA',
+    6: 'TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5',
+    7: 'TLS_RSA_WITH_IDEA_CBC_SHA',
+    8: 'TLS_RSA_EXPORT_WITH_DES40_CBC_SHA',
+    9: 'TLS_RSA_WITH_DES_CBC_SHA',
+    10: 'TLS_RSA_WITH_3DES_EDE_CBC_SHA',
+    11: 'TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA',
+    12: 'TLS_DH_DSS_WITH_DES_CBC_SHA',
+    13: 'TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA',
+    14: 'TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA',
+    15: 'TLS_DH_RSA_WITH_DES_CBC_SHA',
+    16: 'TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA',
+    17: 'TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA',
+    18: 'TLS_DHE_DSS_WITH_DES_CBC_SHA',
+    19: 'TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA',
+    20: 'TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA',
+    21: 'TLS_DHE_RSA_WITH_DES_CBC_SHA',
+    22: 'TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA',
+    23: 'TLS_DH_anon_EXPORT_WITH_RC4_40_MD5',
+    24: 'TLS_DH_anon_WITH_RC4_128_MD5',
+    25: 'TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA',
+    26: 'TLS_DH_anon_WITH_DES_CBC_SHA',
+    27: 'TLS_DH_anon_WITH_3DES_EDE_CBC_SHA',
+    28: 'SSL_FORTEZZA_KEA_WITH_NULL_SHA',
+    29: 'SSL_FORTEZZA_KEA_WITH_FORTEZZA_CBC_SHA',
+    30: 'TLS_KRB5_WITH_DES_CBC_SHA',
+    31: 'TLS_KRB5_WITH_3DES_EDE_CBC_SHA',
+    32: 'TLS_KRB5_WITH_RC4_128_SHA',
+    33: 'TLS_KRB5_WITH_IDEA_CBC_SHA',
+    34: 'TLS_KRB5_WITH_DES_CBC_MD5',
+    35: 'TLS_KRB5_WITH_3DES_EDE_CBC_MD5',
+    36: 'TLS_KRB5_WITH_RC4_128_MD5',
+    37: 'TLS_KRB5_WITH_IDEA_CBC_MD5',
+    38: 'TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA',
+    39: 'TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA',
+    40: 'TLS_KRB5_EXPORT_WITH_RC4_40_SHA',
+    41: 'TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5',
+    42: 'TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5',
+    43: 'TLS_KRB5_EXPORT_WITH_RC4_40_MD5',
+    44: 'TLS_PSK_WITH_NULL_SHA',
+    45: 'TLS_DHE_PSK_WITH_NULL_SHA',
+    46: 'TLS_RSA_PSK_WITH_NULL_SHA',
+    47: 'TLS_RSA_WITH_AES_128_CBC_SHA',
+    48: 'TLS_DH_DSS_WITH_AES_128_CBC_SHA',
+    49: 'TLS_DH_RSA_WITH_AES_128_CBC_SHA',
+    50: 'TLS_DHE_DSS_WITH_AES_128_CBC_SHA',
+    51: 'TLS_DHE_RSA_WITH_AES_128_CBC_SHA',
+    52: 'TLS_DH_anon_WITH_AES_128_CBC_SHA',
+    53: 'TLS_RSA_WITH_AES_256_CBC_SHA',
+    54: 'TLS_DH_DSS_WITH_AES_256_CBC_SHA',
+    55: 'TLS_DH_RSA_WITH_AES_256_CBC_SHA',
+    56: 'TLS_DHE_DSS_WITH_AES_256_CBC_SHA',
+    57: 'TLS_DHE_RSA_WITH_AES_256_CBC_SHA',
+    58: 'TLS_DH_anon_WITH_AES_256_CBC_SHA',
+    59: 'TLS_RSA_WITH_NULL_SHA256',
+    60: 'TLS_RSA_WITH_AES_128_CBC_SHA256',
+    61: 'TLS_RSA_WITH_AES_256_CBC_SHA256',
+    62: 'TLS_DH_DSS_WITH_AES_128_CBC_SHA256',
+    63: 'TLS_DH_RSA_WITH_AES_128_CBC_SHA256',
+    64: 'TLS_DHE_DSS_WITH_AES_128_CBC_SHA256',
+    65: 'TLS_RSA_WITH_CAMELLIA_128_CBC_SHA',
+    66: 'TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA',
+    67: 'TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA',
+    68: 'TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA',
+    69: 'TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA',
+    70: 'TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA',
+    96: 'TLS_RSA_EXPORT1024_WITH_RC4_56_MD5',
+    97: 'TLS_RSA_EXPORT1024_WITH_RC2_CBC_56_MD5',
+    98: 'TLS_RSA_EXPORT1024_WITH_DES_CBC_SHA',
+    99: 'TLS_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA',
+    100: 'TLS_RSA_EXPORT1024_WITH_RC4_56_SHA',
+    101: 'TLS_DHE_DSS_EXPORT1024_WITH_RC4_56_SHA',
+    102: 'TLS_DHE_DSS_WITH_RC4_128_SHA',
+    103: 'TLS_DHE_RSA_WITH_AES_128_CBC_SHA256',
+    104: 'TLS_DH_DSS_WITH_AES_256_CBC_SHA256',
+    105: 'TLS_DH_RSA_WITH_AES_256_CBC_SHA256',
+    106: 'TLS_DHE_DSS_WITH_AES_256_CBC_SHA256',
+    107: 'TLS_DHE_RSA_WITH_AES_256_CBC_SHA256',
+    108: 'TLS_DH_anon_WITH_AES_128_CBC_SHA256',
+    109: 'TLS_DH_anon_WITH_AES_256_CBC_SHA256',
+    128: 'TLS_GOSTR341094_WITH_28147_CNT_IMIT',
+    129: 'TLS_GOSTR341001_WITH_28147_CNT_IMIT',
+    130: 'TLS_GOSTR341094_WITH_NULL_GOSTR3411',
+    131: 'TLS_GOSTR341001_WITH_NULL_GOSTR3411',
+    132: 'TLS_RSA_WITH_CAMELLIA_256_CBC_SHA',
+    133: 'TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA',
+    134: 'TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA',
+    135: 'TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA',
+    136: 'TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA',
+    137: 'TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA',
+    138: 'TLS_PSK_WITH_RC4_128_SHA',
+    139: 'TLS_PSK_WITH_3DES_EDE_CBC_SHA',
+    140: 'TLS_PSK_WITH_AES_128_CBC_SHA',
+    141: 'TLS_PSK_WITH_AES_256_CBC_SHA',
+    142: 'TLS_DHE_PSK_WITH_RC4_128_SHA',
+    143: 'TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA',
+    144: 'TLS_DHE_PSK_WITH_AES_128_CBC_SHA',
+    145: 'TLS_DHE_PSK_WITH_AES_256_CBC_SHA',
+    146: 'TLS_RSA_PSK_WITH_RC4_128_SHA',
+    147: 'TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA',
+    148: 'TLS_RSA_PSK_WITH_AES_128_CBC_SHA',
+    149: 'TLS_RSA_PSK_WITH_AES_256_CBC_SHA',
+    150: 'TLS_RSA_WITH_SEED_CBC_SHA',
+    151: 'TLS_DH_DSS_WITH_SEED_CBC_SHA',
+    152: 'TLS_DH_RSA_WITH_SEED_CBC_SHA',
+    153: 'TLS_DHE_DSS_WITH_SEED_CBC_SHA',
+    154: 'TLS_DHE_RSA_WITH_SEED_CBC_SHA',
+    155: 'TLS_DH_anon_WITH_SEED_CBC_SHA',
+    156: 'TLS_RSA_WITH_AES_128_GCM_SHA256',
+    157: 'TLS_RSA_WITH_AES_256_GCM_SHA384',
+    158: 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256',
+    159: 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384',
+    160: 'TLS_DH_RSA_WITH_AES_128_GCM_SHA256',
+    161: 'TLS_DH_RSA_WITH_AES_256_GCM_SHA384',
+    162: 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256',
+    163: 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384',
+    164: 'TLS_DH_DSS_WITH_AES_128_GCM_SHA256',
+    165: 'TLS_DH_DSS_WITH_AES_256_GCM_SHA384',
+    166: 'TLS_DH_anon_WITH_AES_128_GCM_SHA256',
+    167: 'TLS_DH_anon_WITH_AES_256_GCM_SHA384',
+    168: 'TLS_PSK_WITH_AES_128_GCM_SHA256',
+    169: 'TLS_PSK_WITH_AES_256_GCM_SHA384',
+    170: 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256',
+    171: 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384',
+    172: 'TLS_RSA_PSK_WITH_AES_128_GCM_SHA256',
+    173: 'TLS_RSA_PSK_WITH_AES_256_GCM_SHA384',
+    174: 'TLS_PSK_WITH_AES_128_CBC_SHA256',
+    175: 'TLS_PSK_WITH_AES_256_CBC_SHA384',
+    176: 'TLS_PSK_WITH_NULL_SHA256',
+    177: 'TLS_PSK_WITH_NULL_SHA384',
+    178: 'TLS_DHE_PSK_WITH_AES_128_CBC_SHA256',
+    179: 'TLS_DHE_PSK_WITH_AES_256_CBC_SHA384',
+    180: 'TLS_DHE_PSK_WITH_NULL_SHA256',
+    181: 'TLS_DHE_PSK_WITH_NULL_SHA384',
+    182: 'TLS_RSA_PSK_WITH_AES_128_CBC_SHA256',
+    183: 'TLS_RSA_PSK_WITH_AES_256_CBC_SHA384',
+    184: 'TLS_RSA_PSK_WITH_NULL_SHA256',
+    185: 'TLS_RSA_PSK_WITH_NULL_SHA384',
+    186: 'TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256',
+    187: 'TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256',
+    188: 'TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256',
+    189: 'TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256',
+    190: 'TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256',
+    191: 'TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256',
+    192: 'TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256',
+    193: 'TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256',
+    194: 'TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256',
+    195: 'TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256',
+    196: 'TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256',
+    197: 'TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256',
+    198: 'TLS_SM4_GCM_SM3',
+    199: 'TLS_SM4_CCM_SM3',
+    255: 'TLS_EMPTY_RENEGOTIATION_INFO_SCSV',
+    2570: 'Reserved (GREASE)',
+    4865: 'TLS_AES_128_GCM_SHA256',
+    4866: 'TLS_AES_256_GCM_SHA384',
+    4867: 'TLS_CHACHA20_POLY1305_SHA256',
+    4868: 'TLS_AES_128_CCM_SHA256',
+    4869: 'TLS_AES_128_CCM_8_SHA256',
+    6682: 'Reserved (GREASE)',
+    10794: 'Reserved (GREASE)',
+    14906: 'Reserved (GREASE)',
+    19018: 'Reserved (GREASE)',
+    22016: 'TLS_FALLBACK_SCSV',
+    23130: 'Reserved (GREASE)',
+    27242: 'Reserved (GREASE)',
+    31354: 'Reserved (GREASE)',
+    35466: 'Reserved (GREASE)',
+    39578: 'Reserved (GREASE)',
+    43690: 'Reserved (GREASE)',
+    47802: 'Reserved (GREASE)',
+    49153: 'TLS_ECDH_ECDSA_WITH_NULL_SHA',
+    49154: 'TLS_ECDH_ECDSA_WITH_RC4_128_SHA',
+    49155: 'TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA',
+    49156: 'TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA',
+    49157: 'TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA',
+    49158: 'TLS_ECDHE_ECDSA_WITH_NULL_SHA',
+    49159: 'TLS_ECDHE_ECDSA_WITH_RC4_128_SHA',
+    49160: 'TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA',
+    49161: 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA',
+    49162: 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA',
+    49163: 'TLS_ECDH_RSA_WITH_NULL_SHA',
+    49164: 'TLS_ECDH_RSA_WITH_RC4_128_SHA',
+    49165: 'TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA',
+    49166: 'TLS_ECDH_RSA_WITH_AES_128_CBC_SHA',
+    49167: 'TLS_ECDH_RSA_WITH_AES_256_CBC_SHA',
+    49168: 'TLS_ECDHE_RSA_WITH_NULL_SHA',
+    49169: 'TLS_ECDHE_RSA_WITH_RC4_128_SHA',
+    49170: 'TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA',
+    49171: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA',
+    49172: 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA',
+    49173: 'TLS_ECDH_anon_WITH_NULL_SHA',
+    49174: 'TLS_ECDH_anon_WITH_RC4_128_SHA',
+    49175: 'TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA',
+    49176: 'TLS_ECDH_anon_WITH_AES_128_CBC_SHA',
+    49177: 'TLS_ECDH_anon_WITH_AES_256_CBC_SHA',
+    49178: 'TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA',
+    49179: 'TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA',
+    49180: 'TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA',
+    49181: 'TLS_SRP_SHA_WITH_AES_128_CBC_SHA',
+    49182: 'TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA',
+    49183: 'TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA',
+    49184: 'TLS_SRP_SHA_WITH_AES_256_CBC_SHA',
+    49185: 'TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA',
+    49186: 'TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA',
+    49187: 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256',
+    49188: 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384',
+    49189: 'TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256',
+    49190: 'TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384',
+    49191: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256',
+    49192: 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384',
+    49193: 'TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256',
+    49194: 'TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384',
+    49195: 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256',
+    49196: 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384',
+    49197: 'TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256',
+    49198: 'TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384',
+    49199: 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256',
+    49200: 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384',
+    49201: 'TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256',
+    49202: 'TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384',
+    49203: 'TLS_ECDHE_PSK_WITH_RC4_128_SHA',
+    49204: 'TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA',
+    49205: 'TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA',
+    49206: 'TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA',
+    49207: 'TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256',
+    49208: 'TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384',
+    49209: 'TLS_ECDHE_PSK_WITH_NULL_SHA',
+    49210: 'TLS_ECDHE_PSK_WITH_NULL_SHA256',
+    49211: 'TLS_ECDHE_PSK_WITH_NULL_SHA384',
+    49212: 'TLS_RSA_WITH_ARIA_128_CBC_SHA256',
+    49213: 'TLS_RSA_WITH_ARIA_256_CBC_SHA384',
+    49214: 'TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256',
+    49215: 'TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384',
+    49216: 'TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256',
+    49217: 'TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384',
+    49218: 'TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256',
+    49219: 'TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384',
+    49220: 'TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256',
+    49221: 'TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384',
+    49222: 'TLS_DH_anon_WITH_ARIA_128_CBC_SHA256',
+    49223: 'TLS_DH_anon_WITH_ARIA_256_CBC_SHA384',
+    49224: 'TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256',
+    49225: 'TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384',
+    49226: 'TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256',
+    49227: 'TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384',
+    49228: 'TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256',
+    49229: 'TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384',
+    49230: 'TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256',
+    49231: 'TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384',
+    49232: 'TLS_RSA_WITH_ARIA_128_GCM_SHA256',
+    49233: 'TLS_RSA_WITH_ARIA_256_GCM_SHA384',
+    49234: 'TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256',
+    49235: 'TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384',
+    49236: 'TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256',
+    49237: 'TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384',
+    49238: 'TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256',
+    49239: 'TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384',
+    49240: 'TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256',
+    49241: 'TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384',
+    49242: 'TLS_DH_anon_WITH_ARIA_128_GCM_SHA256',
+    49243: 'TLS_DH_anon_WITH_ARIA_256_GCM_SHA384',
+    49244: 'TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256',
+    49245: 'TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384',
+    49246: 'TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256',
+    49247: 'TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384',
+    49248: 'TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256',
+    49249: 'TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384',
+    49250: 'TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256',
+    49251: 'TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384',
+    49252: 'TLS_PSK_WITH_ARIA_128_CBC_SHA256',
+    49253: 'TLS_PSK_WITH_ARIA_256_CBC_SHA384',
+    49254: 'TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256',
+    49255: 'TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384',
+    49256: 'TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256',
+    49257: 'TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384',
+    49258: 'TLS_PSK_WITH_ARIA_128_GCM_SHA256',
+    49259: 'TLS_PSK_WITH_ARIA_256_GCM_SHA384',
+    49260: 'TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256',
+    49261: 'TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384',
+    49262: 'TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256',
+    49263: 'TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384',
+    49264: 'TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256',
+    49265: 'TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384',
+    49266: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256',
+    49267: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384',
+    49268: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256',
+    49269: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384',
+    49270: 'TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256',
+    49271: 'TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384',
+    49272: 'TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256',
+    49273: 'TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384',
+    49274: 'TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49275: 'TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49276: 'TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49277: 'TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49278: 'TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49279: 'TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49280: 'TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256',
+    49281: 'TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384',
+    49282: 'TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256',
+    49283: 'TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384',
+    49284: 'TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256',
+    49285: 'TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384',
+    49286: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49287: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49288: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49289: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49290: 'TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49291: 'TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49292: 'TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256',
+    49293: 'TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384',
+    49294: 'TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256',
+    49295: 'TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384',
+    49296: 'TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256',
+    49297: 'TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384',
+    49298: 'TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256',
+    49299: 'TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384',
+    49300: 'TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256',
+    49301: 'TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384',
+    49302: 'TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256',
+    49303: 'TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384',
+    49304: 'TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256',
+    49305: 'TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384',
+    49306: 'TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256',
+    49307: 'TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384',
+    49308: 'TLS_RSA_WITH_AES_128_CCM',
+    49309: 'TLS_RSA_WITH_AES_256_CCM',
+    49310: 'TLS_DHE_RSA_WITH_AES_128_CCM',
+    49311: 'TLS_DHE_RSA_WITH_AES_256_CCM',
+    49312: 'TLS_RSA_WITH_AES_128_CCM_8',
+    49313: 'TLS_RSA_WITH_AES_256_CCM_8',
+    49314: 'TLS_DHE_RSA_WITH_AES_128_CCM_8',
+    49315: 'TLS_DHE_RSA_WITH_AES_256_CCM_8',
+    49316: 'TLS_PSK_WITH_AES_128_CCM',
+    49317: 'TLS_PSK_WITH_AES_256_CCM',
+    49318: 'TLS_DHE_PSK_WITH_AES_128_CCM',
+    49319: 'TLS_DHE_PSK_WITH_AES_256_CCM',
+    49320: 'TLS_PSK_WITH_AES_128_CCM_8',
+    49321: 'TLS_PSK_WITH_AES_256_CCM_8',
+    49322: 'TLS_PSK_DHE_WITH_AES_128_CCM_8',
+    49323: 'TLS_PSK_DHE_WITH_AES_256_CCM_8',
+    49324: 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM',
+    49325: 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM',
+    49326: 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8',
+    49327: 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8',
+    49328: 'TLS_ECCPWD_WITH_AES_128_GCM_SHA256',
+    49329: 'TLS_ECCPWD_WITH_AES_256_GCM_SHA384',
+    49330: 'TLS_ECCPWD_WITH_AES_128_CCM_SHA256',
+    49331: 'TLS_ECCPWD_WITH_AES_256_CCM_SHA384',
+    49332: 'TLS_SHA256_SHA256',
+    49333: 'TLS_SHA384_SHA384',
+    49407: 'TLS_ECJPAKE_WITH_AES_128_CCM_8',
+    49408: 'TLS_GOSTR341112_256_WITH_KUZNYECHIK_CTR_OMAC',
+    49409: 'TLS_GOSTR341112_256_WITH_MAGMA_CTR_OMAC',
+    49410: 'TLS_GOSTR341112_256_WITH_28147_CNT_IMIT',
+    49411: 'TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_L',
+    49412: 'TLS_GOSTR341112_256_WITH_MAGMA_MGM_L',
+    49413: 'TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_S',
+    49414: 'TLS_GOSTR341112_256_WITH_MAGMA_MGM_S',
+    51914: 'Reserved (GREASE)',
+    52243: 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256',
+    52244: 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256',
+    52245: 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256',
+    52392: 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256',
+    52393: 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256',
+    52394: 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256',
+    52395: 'TLS_PSK_WITH_CHACHA20_POLY1305_SHA256',
+    52396: 'TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256',
+    52397: 'TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256',
+    52398: 'TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256',
+    53249: 'TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256',
+    53250: 'TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384',
+    53251: 'TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256',
+    53253: 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256',
+    56026: 'Reserved (GREASE)',
+    57345: 'ECDHE_SM1_SM3',
+    57347: 'ECC_SM1_SM3',
+    57349: 'IBSDH_SM1_SM3',
+    57351: 'IBC_SM1_SM3',
+    57353: 'RSA_SM1_SM3',
+    57354: 'RSA_SM1_SHA1',
+    57361: 'ECDHE_SM4_CBC_SM3',
+    57363: 'ECC_SM4_CBC_SM3',
+    57365: 'IBSDH_SM4_CBC_SM3',
+    57367: 'IBC_SM4_CBC_SM3',
+    57369: 'RSA_SM4_CBC_SM3',
+    57370: 'RSA_SM4_CBC_SHA1',
+    57372: 'RSA_SM4_CBC_SHA256',
+    57425: 'ECDHE_SM4_GCM_SM3',
+    57427: 'ECC_SM4_GCM_SM3',
+    57429: 'IBSDH_SM4_GCM_SM3',
+    57431: 'IBC_SM4_GCM_SM3',
+    57433: 'RSA_SM4_GCM_SM3',
+    57434: 'RSA_SM4_GCM_SHA256',
+    58384: 'TLS_RSA_WITH_ESTREAM_SALSA20_SHA1',
+    58385: 'TLS_RSA_WITH_SALSA20_SHA1',
+    58386: 'TLS_ECDHE_RSA_WITH_ESTREAM_SALSA20_SHA1',
+    58387: 'TLS_ECDHE_RSA_WITH_SALSA20_SHA1',
+    58388: 'TLS_ECDHE_ECDSA_WITH_ESTREAM_SALSA20_SHA1',
+    58389: 'TLS_ECDHE_ECDSA_WITH_SALSA20_SHA1',
+    58390: 'TLS_PSK_WITH_ESTREAM_SALSA20_SHA1',
+    58391: 'TLS_PSK_WITH_SALSA20_SHA1',
+    58392: 'TLS_ECDHE_PSK_WITH_ESTREAM_SALSA20_SHA1',
+    58393: 'TLS_ECDHE_PSK_WITH_SALSA20_SHA1',
+    58394: 'TLS_RSA_PSK_WITH_ESTREAM_SALSA20_SHA1',
+    58395: 'TLS_RSA_PSK_WITH_SALSA20_SHA1',
+    58396: 'TLS_DHE_PSK_WITH_ESTREAM_SALSA20_SHA1',
+    58397: 'TLS_DHE_PSK_WITH_SALSA20_SHA1',
+    58398: 'TLS_DHE_RSA_WITH_ESTREAM_SALSA20_SHA1',
+    58399: 'TLS_DHE_RSA_WITH_SALSA20_SHA1',
+    60138: 'Reserved (GREASE)',
+    64250: 'Reserved (GREASE)',
+    65278: 'SSL_RSA_FIPS_WITH_DES_CBC_SHA',
+    65279: 'SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA',
+    65504: 'SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA',
+    65505: 'SSL_RSA_FIPS_WITH_DES_CBC_SHA',
+}
+
+TLS_VERSIONS: dict[int, str] = {
+    0x0002: "SSL 2.0", 0x0300: "SSL 3.0", 0x0301: "TLS 1.0", 0x0302: "TLS 1.1",
+    0x0303: "TLS 1.2", 0x0304: "TLS 1.3",
+}
+
+# GREASE values (RFC 8701) are random placeholders and never a real cipher or version.
+
+def is_grease(value: int) -> bool:
+    return (value & 0x0F0F) == 0x0A0A and ((value >> 8) & 0xFF) == (value & 0xFF)
+
+def parse_int(raw) -> int | None:
+    """Parse tshark numeric output given as decimal or 0x-prefixed hex."""
+    if raw is None:
+        return None
+    s = str(raw).strip()
+    if not s:
+        return None
+    try:
+        return int(s, 16) if s.lower().startswith("0x") else int(s)
+    except ValueError:
+        return None
+
+def cipher_name(raw) -> str | None:
+    v = parse_int(raw)
+    if v is None:
+        return str(raw) if raw else None
+    return CIPHER_SUITES.get(v, f"0x{v:04x}")
+
+def version_name(raw) -> str | None:
+    v = parse_int(raw)
+    if v is None:
+        return str(raw) if raw else None
+    if 0x7F00 <= v <= 0x7FFF:
+        return "TLS 1.3 (draft)"
+    return TLS_VERSIONS.get(v, f"0x{v:04x}")
+
+# Cipher-suite properties used by the TLS detector.
+
+def cipher_is_weak(name: str) -> str | None:
+    """Return a reason when the suite is considered broken or obsolete, else None."""
+    n = (name or "").upper()
+    if not n or n.startswith("0X"):
+        return None
+    if "_NULL_" in n or n.endswith("_NULL") or "WITH_NULL" in n:
+        return "no encryption (NULL cipher)"
+    if "ANON" in n:
+        return "anonymous key exchange (no authentication)"
+    if "EXPORT" in n:
+        return "export-grade key length"
+    if "RC4" in n:
+        return "RC4 stream cipher"
+    if "_DES_" in n or "DES40" in n or "DES_CBC" in n and "3DES" not in n:
+        return "single DES"
+    if "3DES" in n:
+        return "3DES (Sweet32)"
+    if "IDEA" in n or "SEED" in n or "ARIA" in n and "GCM" not in n:
+        return "legacy block cipher"
+    if n.endswith("_MD5") or "_MD5" in n:
+        return "MD5 MAC"
+    return None
+
+def cipher_is_legacy(name: str) -> str | None:
+    """Return a reason when the suite lacks forward secrecy or an AEAD mode."""
+    n = (name or "").upper()
+    if not n or n.startswith("0X") or n.startswith("TLS_AES_") or n.startswith("TLS_CHACHA20_"):
+        return None
+    if cipher_is_weak(n):
+        return None
+    reasons = []
+    if not ("ECDHE" in n or "DHE" in n):
+        reasons.append("no forward secrecy (static key exchange)")
+    if "CBC" in n or n.endswith("_SHA") or n.endswith("_SHA256") and "GCM" not in n and "CCM" not in n and "CHACHA" not in n:
+        reasons.append("CBC / non-AEAD mode")
+    return "; ".join(reasons) or None

docker-compose.yml

@@ -22,6 +22,10 @@ services:
       DJANGO_SETTINGS_MODULE: IoTPreTester.settings
       DATABASE_PATH: /app/data/db.sqlite3
       CAPTURE_UPLOAD_DIR: /app/files/uploads
+      CAPTURE_PACKET_LIMIT: ${CAPTURE_PACKET_LIMIT:-0}
+      CAPTURE_MAX_TEXT_PAYLOADS: ${CAPTURE_MAX_TEXT_PAYLOADS:-50000}
+      CAPTURE_MAX_PAYLOAD_CHARS: ${CAPTURE_MAX_PAYLOAD_CHARS:-16384}
+      ANALYZER_BENIGN_DOMAINS: ${ANALYZER_BENIGN_DOMAINS:-}
       CORS_ALLOW_ALL_ORIGINS: ${CORS_ALLOW_ALL_ORIGINS:-False}
       CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000}
       DJANGO_SUPERUSER_USERNAME: ${DJANGO_SUPERUSER_USERNAME:-admin}
@@ -31,6 +35,7 @@ services:
       GUNICORN_WORKERS: ${GUNICORN_WORKERS:-4}
       GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-600}
       LOG_LEVEL: ${LOG_LEVEL:-info}
+      LOG_DIR: /app/logs
     networks:
       - demo-network
     healthcheck:

docker/entrypoint-backend.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
 set -eu
 
-echo "Starting IoT PreTester 4.6 Demo backend"
+echo "Starting IoT PreTester 4.6 Demo backend (LOG_LEVEL=${LOG_LEVEL:-info})"
+echo "tshark: $(tshark -v 2>/dev/null | head -n 1 || echo 'NOT FOUND')"
 
 mkdir -p /app/data /app/files/uploads /app/logs /app/django_project/staticfiles
 cd /app/django_project
@@ -32,14 +33,29 @@ user.save()
 print("Demo administrator created" if created else "Demo administrator synchronized")
 PY
 
+# A restart loses in-flight background analyses. Without this the frontend
+# would poll a capture that is stuck in "analyzing" forever.
+python manage.py shell <<'PY'
+from IoTAnalyzer.models import File
+
+stale = File.objects.filter(status__in=["analyzing", "pending"])
+count = stale.count()
+if count:
+    stale.update(status="analysis_failed")
+print(f"Reset {count} interrupted capture analyses")
+PY
+
 if [ "${INIT_DATABASE:-true}" = "true" ]; then
     python create_models.py
 fi
 
+# Errors and application logs go to stdout/stderr so `docker compose logs`
+# shows them; /app/logs keeps rotating copies (backend.log, access.log).
 exec gunicorn IoTPreTester.wsgi:application \
     --bind 0.0.0.0:8000 \
     --workers "${GUNICORN_WORKERS:-4}" \
     --timeout "${GUNICORN_TIMEOUT:-120}" \
     --access-logfile /app/logs/access.log \
-    --error-logfile /app/logs/error.log \
+    --error-logfile - \
+    --capture-output \
     --log-level "${LOG_LEVEL:-info}"

frontend/src/app/error.tsx

@@ -0,0 +1,25 @@
+"use client"
+
+import { useEffect } from "react"
+import { Button } from "@/components/ui/button"
+import { getLogger, describeError } from "@/lib/logger"
+
+const log = getLogger("route")
+
+export default function RouteError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) {
+  useEffect(() => {
+    log.error(`Render failure on ${typeof window !== "undefined" ? window.location.pathname : "?"}: ${describeError(error)}`,
+      { digest: error.digest, stack: error.stack })
+  }, [error])
+  return (
+    <div className="mx-auto max-w-2xl p-8">
+      <h1 className="text-xl font-semibold">Something went wrong</h1>
+      <p className="mt-2 text-sm text-muted-foreground">{describeError(error)}</p>
+      {error.digest && <p className="mt-1 text-xs text-muted-foreground">Digest: {error.digest}</p>}
+      <p className="mt-3 text-xs text-muted-foreground">
+        Details are in the browser console; run <code>window.__iotpDumpLogs()</code> to copy the client log.
+      </p>
+      <Button className="mt-4" onClick={() => reset()}>Try again</Button>
+    </div>
+  )
+}

frontend/src/app/findings/page.tsx

@@ -7,6 +7,11 @@ import { ArrowLeft, FileJson, FileText, FileCode, ShieldAlert, Globe, ChevronRig
 import { ProtectedRoute } from "@/components/protected-route"
 import { Button } from "@/components/ui/button"
 import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"
+import { InventoryPanel } from "@/components/inventory-panel"
+import { apiFetch, apiErrorMessage } from "@/lib/api"
+import { getLogger } from "@/lib/logger"
+
+const log = getLogger("findings")
 
 // API types mirror the backend finding schema.
 interface Finding {
@@ -36,12 +41,30 @@ interface Summary {
   by_category: Record<string, number>
   highlights: Finding[]
   top: Finding[]
+  device_related?: number
+  other_hosts?: number
 }
 
 interface AnalysisResult {
   meta?: Record<string, unknown>
   summary: Summary
   findings: Finding[]
+  // Complete observation inventory (hosts, endpoints, TLS, DNS, coverage).
+  inventory?: Record<string, unknown>
+  // Present while the capture analysis has not produced a report yet.
+  status?: "analyzing" | "failed" | "empty" | "ready"
+  capture_status?: string
+  message?: string
+  // Failure reason recorded by the backend and whether a per-capture log exists.
+  error?: string | null
+  log_available?: boolean
+}
+
+interface AnalysisLog {
+  available: boolean
+  log: string
+  path?: string
+  message?: string
 }
 
 interface DeepStatus {
@@ -122,6 +145,7 @@ function affectedLine(f: Finding): string {
 // Evidence rendered in dedicated sections is excluded from raw details.
 const EVIDENCE_SKIP = new Set([
   "dedup_key", "sources", "artifact", "capture_file", "source_packet_count", "occurrences",
+  "device_related",
 ])
 
 type SourceRef = {
@@ -172,47 +196,57 @@ function FindingsContent() {
   const [status, setStatus] = useState<DeepStatus | null>(null)
   const [cloud, setCloud] = useState<CloudResults | null>(null)
   const [analyzing, setAnalyzing] = useState(false)
+  const [captureRunning, setCaptureRunning] = useState(false)
   const [selUrls, setSelUrls] = useState<Set<number>>(new Set())
   const [selHashes, setSelHashes] = useState<Set<number>>(new Set())
+  const [backendLog, setBackendLog] = useState<AnalysisLog | null>(null)
+  const [logLoading, setLogLoading] = useState(false)
 
   const fetchCloud = async () => {
     if (!fileId) return
     try {
-      const res = await fetch(getApiUrl(`get_cloud_results/${fileId}`), {
-        cache: "no-store",
-        credentials: "include",
-      })
-      if (res.ok) setCloud(await res.json())
-    } catch { /* Optional enrichment data does not block the finding view. */ }
+      setCloud(await apiFetch<CloudResults>(`get_cloud_results/${fileId}`))
+    } catch (e) {
+      log.debug(`Cloud results unavailable: ${apiErrorMessage(e)}`)
+    }
+  }
+
+  const fetchBackendLog = async () => {
+    if (!fileId) return
+    setLogLoading(true)
+    try {
+      setBackendLog(await apiFetch<AnalysisLog>(`get_analysis_log/${fileId}?lines=400`))
+    } catch (e) {
+      setBackendLog({ available: false, log: "", message: apiErrorMessage(e) })
+    } finally {
+      setLogLoading(false)
+    }
   }
 
   useEffect(() => {
     if (!fileId) { setError("No capture ID was provided."); setLoading(false); return }
     const load = async () => {
       try {
-        const res = await fetch(getApiUrl(`get_findings/${fileId}`), {
-          cache: "no-store",
-          credentials: "include",
-        })
-        if (!res.ok) throw new Error(`HTTP ${res.status}`)
-        setData(await res.json())
+        const result = await apiFetch<AnalysisResult>(`get_findings/${fileId}`)
+        setData(result)
+        setCaptureRunning(result.status === "analyzing")
+        log.info(`Capture ${fileId}: status=${result.status} findings=${result.findings?.length ?? 0}`,
+          result.error ? { error: result.error } : undefined)
+        if (result.status === "failed") fetchBackendLog()
       } catch (e) {
-        setError(`Findings could not be loaded: ${e instanceof Error ? e.message : String(e)}`)
+        log.error(`Findings for capture ${fileId} could not be loaded: ${apiErrorMessage(e)}`)
+        setError(`Findings could not be loaded: ${apiErrorMessage(e)}`)
       } finally {
         setLoading(false)
       }
       await fetchCloud()
       try {
-        const sres = await fetch(getApiUrl(`get_analysis_status/${fileId}`), {
-          cache: "no-store",
-          credentials: "include",
-        })
-        if (sres.ok) {
-          const s: DeepStatus = await sres.json()
-          setStatus(s)
-          if (s.state === "running") setAnalyzing(true)
-        }
-      } catch { /* Status is optional until enrichment starts. */ }
+        const s = await apiFetch<DeepStatus>(`get_analysis_status/${fileId}`)
+        setStatus(s)
+        if (s.state === "running") setAnalyzing(true)
+      } catch (e) {
+        log.debug(`Enrichment status unavailable: ${apiErrorMessage(e)}`)
+      }
     }
     load()
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -237,16 +271,15 @@ function FindingsContent() {
     let timer: ReturnType<typeof setInterval> | null = null
     const poll = async () => {
       try {
-        const res = await fetch(getApiUrl(`get_analysis_status/${fileId}`), {
-          cache: "no-store",
-          credentials: "include",
-        })
-        if (res.ok) {
-          const s: DeepStatus = await res.json()
-          setStatus(s)
-          if (s.state === "done" || s.state === "error") setAnalyzing(false)
+        const s = await apiFetch<DeepStatus>(`get_analysis_status/${fileId}`)
+        setStatus(s)
+        if (s.state === "done" || s.state === "error") {
+          setAnalyzing(false)
+          log.info(`Enrichment for capture ${fileId} finished: ${s.state} (${s.message})`)
         }
-      } catch { /* Retry on the next polling interval. */ }
+      } catch (e) {
+        log.warn(`Enrichment status poll failed: ${apiErrorMessage(e)}`)
+      }
       await fetchCloud()
     }
     poll()
@@ -255,20 +288,42 @@ function FindingsContent() {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [fileId, analyzing])
 
+  // Poll while the capture analysis itself is still running.
+  useEffect(() => {
+    if (!fileId || !captureRunning) return
+    let timer: ReturnType<typeof setInterval> | null = null
+    const poll = async () => {
+      try {
+        const result = await apiFetch<AnalysisResult>(`get_findings/${fileId}`)
+        setData(result)
+        if (result.status !== "analyzing") {
+          setCaptureRunning(false)
+          log.info(`Capture ${fileId} analysis finished: ${result.status}`, result.error ? { error: result.error } : undefined)
+          if (result.status === "failed") fetchBackendLog()
+          await fetchCloud()
+        }
+      } catch (e) {
+        log.warn(`Capture status poll failed: ${apiErrorMessage(e)}`)
+      }
+    }
+    timer = setInterval(poll, 5000)
+    return () => { if (timer) clearInterval(timer) }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [fileId, captureRunning])
+
   const startAnalysis = async () => {
     if (!fileId || (selUrls.size + selHashes.size) === 0) return
     try {
-      const response = await fetch(getApiUrl(`start_analysis/${fileId}`), {
+      await apiFetch(`start_analysis/${fileId}`, {
         method: "POST",
-        credentials: "include",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ url_ids: Array.from(selUrls), hash_ids: Array.from(selHashes) }),
-        cache: "no-store",
       })
-      if (!response.ok) throw new Error(`HTTP ${response.status}`)
+      log.info(`Enrichment started for capture ${fileId}: ${selUrls.size} URLs, ${selHashes.size} hashes`)
       setAnalyzing(true)
     } catch (e) {
-      setError(`Enrichment could not be started: ${e instanceof Error ? e.message : String(e)}`)
+      log.error(`Enrichment could not be started: ${apiErrorMessage(e)}`)
+      setError(`Enrichment could not be started: ${apiErrorMessage(e)}`)
     }
   }
 
@@ -333,6 +388,65 @@ function FindingsContent() {
             </CardDescription>
           </CardHeader>
           <CardContent>
+            {/* Status of the capture analysis itself. */}
+            {data?.status && data.status !== "ready" && (
+              <div className={`mb-6 rounded-lg border p-4 ${
+                data.status === "analyzing" ? "border-blue-300 bg-blue-50"
+                : data.status === "failed" ? "border-red-300 bg-red-50"
+                : "border-gray-300 bg-gray-50"
+              }`}>
+                <p className="font-medium">
+                  {data.status === "analyzing" && "⏳ The capture is being analyzed…"}
+                  {data.status === "failed" && "⚠️ The capture analysis failed"}
+                  {data.status === "empty" && "No security analysis is available for this capture."}
+                </p>
+                <p className="text-sm text-muted-foreground mt-1">
+                  {data.status === "analyzing"
+                    ? "Large captures take a moment. This page refreshes automatically."
+                    : data.message}
+                </p>
+                {data.status === "failed" && (
+                  <div className="mt-3 space-y-2">
+                    {data.error && (
+                      <div className="rounded-md border border-red-200 bg-white/70 p-2">
+                        <div className="text-xs font-medium text-red-800">Error</div>
+                        <code className="text-xs break-all whitespace-pre-wrap">{data.error}</code>
+                      </div>
+                    )}
+                    <div className="flex items-center gap-2 flex-wrap">
+                      <Button size="sm" variant="outline" onClick={fetchBackendLog} disabled={logLoading}>
+                        {logLoading ? "Loading log…" : backendLog ? "Refresh analysis log" : "Show analysis log"}
+                      </Button>
+                      {backendLog?.available && backendLog.path && (
+                        <span className="text-xs text-muted-foreground">Backend file: {backendLog.path}</span>
+                      )}
+                    </div>
+                    {backendLog && (
+                      backendLog.available ? (
+                        <pre className="max-h-96 overflow-auto rounded-md bg-slate-900 p-3 text-[11px] leading-snug text-slate-100 whitespace-pre-wrap break-all">{backendLog.log || "(empty log)"}</pre>
+                      ) : (
+                        <p className="text-xs text-muted-foreground">{backendLog.message || "No analysis log is available."}</p>
+                      )
+                    )}
+                    <p className="text-xs text-muted-foreground">
+                      Container log: <code>docker compose logs backend</code> · rotating file: <code>/app/logs/backend.log</code> ·
+                      client log: <code>window.__iotpDumpLogs()</code> in the browser console.
+                    </p>
+                  </div>
+                )}
+              </div>
+            )}
+
+            {/* A partial pass must be visible as such. */}
+            {Boolean(data?.meta?.truncated) && (
+              <div className="mb-6 rounded-lg border border-amber-300 bg-amber-50 p-4">
+                <p className="font-medium">⚠️ Incomplete analysis</p>
+                <p className="text-sm text-muted-foreground mt-1">
+                  {String(data?.meta?.truncation_reason ?? "Configured analysis limits were reached.")}
+                </p>
+              </div>
+            )}
+
             {/* Status of optional URL and hash enrichment. */}
             {status && status.state !== "pending" && (
               <div className={`mb-6 rounded-lg border p-4 ${
@@ -380,6 +494,20 @@ function FindingsContent() {
               </div>
             )}
 
+            {/* Device under test */}
+            {Boolean(data?.meta?.device_ip) && (
+              <div className="mb-6 rounded-lg border border-emerald-300 bg-emerald-50 dark:bg-emerald-950/30 p-4 text-sm">
+                <span className="font-medium">Device under test:</span>{" "}
+                <code>{String(data?.meta?.device_ip)}</code>
+                {Array.isArray(data?.meta?.device_macs) && (data?.meta?.device_macs as string[]).length > 0 && (
+                  <> (MAC {(data?.meta?.device_macs as string[]).join(", ")})</>
+                )}
+                <span className="text-muted-foreground">
+                  {" "}· device-related findings: {summary?.device_related ?? 0} · other hosts: {summary?.other_hosts ?? 0}
+                </span>
+              </div>
+            )}
+
             {/* Prioritized highlights */}
             {summary && summary.highlights?.length > 0 && (
               <div className="mb-6 rounded-lg border border-red-300 bg-red-50 p-4">
@@ -413,7 +541,12 @@ function FindingsContent() {
                   <Card key={f.finding_id} className="border-l-4" style={{ borderLeftColor: sevBorder(f.severity) }}>
                     <CardContent className="p-4 space-y-2">
                       <div className="flex items-center justify-between gap-2 flex-wrap">
-                        <h4 className="font-semibold">{i + 1}. {f.title}</h4>
+                        <h4 className="font-semibold">
+                          {i + 1}. {f.title}
+                          {f.evidence?.device_related === true && (
+                            <span className="ml-2 rounded bg-emerald-600 px-1.5 py-0.5 text-[10px] font-semibold text-white align-middle">DEVICE</span>
+                          )}
+                        </h4>
                         <div className="flex items-center gap-2">
                           <Pill severity={f.severity} />
                           <span className="text-xs text-muted-foreground">Confidence: {f.confidence}</span>
@@ -470,6 +603,8 @@ function FindingsContent() {
               </div>
             )}
 
+            {data?.status === "ready" && <InventoryPanel inventory={data?.inventory} />}
+
             {cloud && (cloud.hashes.length > 0 || cloud.urls.length > 0) && (
               <h2 className="text-2xl font-bold mt-10 mb-3">Extracted artifacts</h2>
             )}

frontend/src/components/file-upload.tsx

@@ -5,6 +5,9 @@ import * as React from "react"
 import { useState } from "react"
 import { useRouter } from "next/navigation"
 import axios from "axios"
+import { getLogger } from "@/lib/logger"
+
+const log = getLogger("upload")
 import { FileIcon, UploadCloudIcon, XIcon, ScanIcon, ArrowLeft } from "lucide-react"
 import { useDropzone } from "react-dropzone"
 import type { FileRejection } from "react-dropzone"
@@ -95,9 +98,10 @@ export function FileUpload({ onBack, deviceId }: FileUploadProps) {
       })
 
       const fileId = response.data.data.id
+      log.info(`Capture uploaded: file_id=${fileId} name=${file.name} size=${file.size} device=${deviceId} ip=${fileDetails.ipAddress}`)
       router.push(`/scan-results?fileId=${fileId}`)
     } catch (err) {
-      console.error("Scan error:", err)
+      log.error(`Upload of ${file.name} (${file.size} bytes) failed`, err)
       const apiError = axios.isAxiosError(err)
         ? err.response?.data?.error || err.response?.data?.capture?.[0] || err.message
         : err instanceof Error

frontend/src/components/inventory-panel.tsx

@@ -0,0 +1,262 @@
+"use client"
+
+// Observation inventory: everything the analysis saw, so that an analyst can
+// verify nothing was overlooked even when no finding was raised.
+
+type Rec = Record<string, unknown>
+
+interface Props {
+  inventory?: Rec | null
+}
+
+function kb(n: unknown): string {
+  const v = typeof n === "number" ? n : Number(n ?? 0)
+  if (!isFinite(v)) return "—"
+  if (v >= 1024 * 1024) return `${(v / (1024 * 1024)).toFixed(1)} MB`
+  if (v >= 1024) return `${(v / 1024).toFixed(1)} kB`
+  return `${v} B`
+}
+
+function list(v: unknown, max = 8): string {
+  if (!Array.isArray(v) || v.length === 0) return "—"
+  const s = v.slice(0, max).map(String).join(", ")
+  return v.length > max ? `${s}, … (+${v.length - max})` : s
+}
+
+function Section({ title, count, open, children }: { title: string; count?: number; open?: boolean; children: React.ReactNode }) {
+  return (
+    <details className="mt-2 rounded-lg border bg-card" open={open}>
+      <summary className="cursor-pointer px-4 py-2 font-medium flex items-center justify-between">
+        <span>{title}</span>
+        {typeof count === "number" && <span className="text-xs text-muted-foreground">{count}</span>}
+      </summary>
+      <div className="px-4 pb-3 text-sm overflow-x-auto">{children}</div>
+    </details>
+  )
+}
+
+function DeviceBadge({ on }: { on: unknown }) {
+  return on ? <span className="ml-1 rounded bg-emerald-600 px-1.5 py-0.5 text-[10px] font-semibold text-white">DEVICE</span> : null
+}
+
+export function InventoryPanel({ inventory }: Props) {
+  if (!inventory || typeof inventory !== "object") return null
+  const cap = (inventory.capture as Rec) || {}
+  const dev = (inventory.device as Rec) || {}
+  const hosts = (inventory.hosts as Rec[]) || []
+  const eps = (inventory.external_endpoints as Rec[]) || []
+  const svcs = (inventory.services as Rec[]) || []
+  const tls = (inventory.tls as Rec) || {}
+  const dns = (inventory.dns as Rec) || {}
+  const http = (inventory.http as Rec) || {}
+  const clear = (inventory.cleartext as Rec) || {}
+  const cov = (inventory.coverage as Rec) || {}
+  const protos = ((inventory.protocols as Rec) || {}).by_protocol as Record<string, number> | undefined
+
+  return (
+    <div className="mt-10">
+      <h2 className="text-2xl font-bold mb-1">Observation inventory</h2>
+      <p className="text-sm text-muted-foreground mb-3">
+        Complete list of what the capture contained — {String(cap.packets ?? 0)} packets, {String(cap.internal_hosts ?? 0)} internal
+        and {String(cap.external_hosts ?? 0)} external hosts, {String(cap.conversations ?? 0)} flows
+        {Array.isArray(cap.vlans) && cap.vlans.length > 0 && <> · VLANs {list(cap.vlans)}</>}
+        {cap.extractor ? <> · {String(cap.extractor)}</> : null}.
+      </p>
+
+      {/* Coverage first: an analyst must know what could not be inspected. */}
+      <div className={`rounded-lg border p-3 text-sm ${cov.complete ? "border-emerald-300 bg-emerald-50 dark:bg-emerald-950/30" : "border-amber-300 bg-amber-50 dark:bg-amber-950/30"}`}>
+        <div className="font-medium">{cov.complete ? "✅ Complete extraction" : "⚠️ Partial extraction"}</div>
+        <div className="text-muted-foreground">
+          {String(cov.flows_total ?? 0)} flows · {String(cov.flows_encrypted ?? 0)} encrypted · {String(cov.flows_undecoded ?? 0)} with undecoded payload ·
+          {" "}{String(cov.text_payloads_scanned ?? 0)} clear-text payloads scanned
+        </div>
+        {Array.isArray(cov.notes) && cov.notes.length > 0 && (
+          <ul className="mt-1 list-disc pl-5 text-muted-foreground">
+            {(cov.notes as string[]).map((n, i) => <li key={i}>{n}</li>)}
+          </ul>
+        )}
+      </div>
+
+      <Section title="Device under test" open>
+        {!dev.configured ? (
+          <div>
+            <p className="text-muted-foreground">No device address was configured for this capture. Likely candidates:</p>
+            <ul className="list-disc pl-5">
+              {((dev.candidates as Rec[]) || []).map((c, i) => (
+                <li key={i}><code>{String(c.ip)}</code> {list(c.names)} · {list(c.macs)} · {String(c.packets)} packets</li>
+              ))}
+            </ul>
+          </div>
+        ) : !dev.seen ? (
+          <p className="text-amber-700">{String(dev.note)}</p>
+        ) : (
+          <div className="space-y-1">
+            <div><code>{String(dev.ip)}</code> · MAC {list(dev.macs)} · names {list(dev.names)}
+              {dev.dhcp_vendor_class ? <> · DHCP vendor class {String(dev.dhcp_vendor_class)}</> : null}</div>
+            <div className="text-muted-foreground">{String(dev.packets)} packets · sent {kb(dev.bytes_sent)} · received {kb(dev.bytes_received)}</div>
+            <div>Services offered: {list(dev.ports_offered)} · ports contacted: {list(dev.ports_contacted, 20)}</div>
+            <div>Protocols: {list(dev.protocols, 20)} · TLS handshakes: {String(dev.tls_sessions ?? 0)} · clear-text flows: {String(dev.cleartext_flows ?? 0)}</div>
+            {Array.isArray(dev.dns_queries) && dev.dns_queries.length > 0 && <div>DNS names queried: {list(dev.dns_queries, 30)}</div>}
+            {Array.isArray(dev.peers) && dev.peers.length > 0 && (
+              <table className="mt-2 w-full text-xs">
+                <thead><tr className="text-left text-muted-foreground"><th>Peer</th><th>Names</th><th>Scope</th><th>Ports</th><th>Protocols</th><th>To peer</th><th>From peer</th></tr></thead>
+                <tbody>
+                  {(dev.peers as Rec[]).map((p, i) => (
+                    <tr key={i} className="border-t">
+                      <td className="font-mono">{String(p.ip)}</td><td>{list(p.names, 3)}</td>
+                      <td>{p.external ? "external" : "internal"}</td><td>{list(p.ports, 6)}</td>
+                      <td>{list(p.protocols, 5)}</td><td>{kb(p.bytes_to_peer)}</td><td>{kb(p.bytes_from_peer)}</td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            )}
+          </div>
+        )}
+      </Section>
+
+      <Section title="Internal hosts" count={hosts.length}>
+        <table className="w-full text-xs">
+          <thead><tr className="text-left text-muted-foreground"><th>IP</th><th>Names</th><th>MAC</th><th>Packets</th><th>Offers</th><th>Ext. endpoints</th><th>Protocols</th></tr></thead>
+          <tbody>
+            {hosts.map((h, i) => (
+              <tr key={i} className="border-t">
+                <td className="font-mono">{String(h.ip)}<DeviceBadge on={h.is_device} /></td><td>{list(h.names, 3)}</td>
+                <td className="font-mono">{list(h.macs, 2)}</td><td>{String(h.packets)}</td><td>{list(h.ports_offered, 8)}</td>
+                <td>{String(h.external_endpoints ?? 0)}</td><td>{list(h.protocols, 6)}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Section>
+
+      <Section title="External endpoints" count={eps.length}>
+        <table className="w-full text-xs">
+          <thead><tr className="text-left text-muted-foreground"><th>IP</th><th>Names</th><th>Ports</th><th>Protocols</th><th>Encrypted</th><th>Out</th><th>In</th><th>Internal hosts</th></tr></thead>
+          <tbody>
+            {eps.map((e, i) => (
+              <tr key={i} className="border-t">
+                <td className="font-mono">{String(e.ip)}<DeviceBadge on={e.device} /></td><td>{list(e.names, 3)}</td>
+                <td>{list(e.ports, 6)}</td><td>{list(e.protocols, 5)}</td>
+                <td>{e.encrypted ? "yes" : <span className="text-amber-700">no / unknown</span>}</td>
+                <td>{kb(e.bytes_out)}</td><td>{kb(e.bytes_in)}</td><td>{list(e.internal_hosts, 3)}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Section>
+
+      <Section title="Services observed" count={svcs.length}>
+        <table className="w-full text-xs">
+          <thead><tr className="text-left text-muted-foreground"><th>Endpoint</th><th>Service</th><th>Transport</th><th>Protocols</th><th>Encryption</th><th>Flows</th><th>Clients</th></tr></thead>
+          <tbody>
+            {svcs.map((s, i) => (
+              <tr key={i} className="border-t">
+                <td className="font-mono">{String(s.ip)}:{String(s.port)}<DeviceBadge on={s.device} /></td>
+                <td>{String(s.service)}</td><td>{String(s.transport)}</td><td>{list(s.protocols, 4)}</td>
+                <td>{s.encrypted ? "encrypted" : s.cleartext_app ? <span className="text-amber-700">clear-text</span> : "unknown"}</td>
+                <td>{String(s.flows)}</td><td>{list(s.clients, 3)}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Section>
+
+      <Section title="TLS posture" count={Number(tls.sessions_total ?? 0)}>
+        <div className="text-xs text-muted-foreground mb-2">
+          Versions: {Object.entries((tls.versions as Record<string, number>) || {}).map(([k, v]) => `${k} ×${v}`).join(", ") || "—"} ·
+          Ciphers: {Object.entries((tls.ciphers as Record<string, number>) || {}).map(([k, v]) => `${k} ×${v}`).join(", ") || "—"}
+        </div>
+        <table className="w-full text-xs">
+          <thead><tr className="text-left text-muted-foreground"><th>Client → Server</th><th>SNI</th><th>Version</th><th>Cipher</th><th>Client max</th><th>Certificate</th><th>Notes</th></tr></thead>
+          <tbody>
+            {((tls.sessions as Rec[]) || []).map((s, i) => {
+              const cert = (s.certificate as Rec) || null
+              return (
+                <tr key={i} className="border-t">
+                  <td className="font-mono">{String(s.client)} → {String(s.server)}:{String(s.port)}<DeviceBadge on={s.device} /></td>
+                  <td>{s.sni ? String(s.sni) : "—"}</td><td>{s.version ? String(s.version) : "no ServerHello"}</td>
+                  <td>{s.cipher ? String(s.cipher) : "—"}</td><td>{s.client_max_version ? String(s.client_max_version) : "—"}</td>
+                  <td>{cert ? `${String(cert.subject ?? "?")} by ${String(cert.issuer ?? "?")}${cert.not_after ? `, until ${String(cert.not_after)}` : ""}` : (s.version === "TLS 1.3" ? "encrypted (TLS 1.3)" : "—")}</td>
+                  <td>{[s.mutual_tls ? "mutual TLS" : null, Array.isArray(s.alerts) && s.alerts.length ? `alerts: ${list(s.alerts, 3)}` : null, Array.isArray(s.alpn) && s.alpn.length ? `ALPN ${list(s.alpn, 2)}` : null].filter(Boolean).join(" · ") || "—"}</td>
+                </tr>
+              )
+            })}
+          </tbody>
+        </table>
+      </Section>
+
+      <Section title="DNS" count={Number(dns.unique_names ?? 0)}>
+        <div className="text-xs text-muted-foreground mb-2">
+          {String(dns.queries ?? 0)} queries · {String(dns.responses ?? 0)} responses · codes {Object.entries((dns.rcodes as Record<string, number>) || {}).map(([k, v]) => `${k} ${v}`).join(", ") || "—"} ·
+          resolvers {Object.entries((dns.resolvers as Record<string, number>) || {}).map(([k, v]) => `${k} (${v})`).join(", ") || "—"}
+        </div>
+        <table className="w-full text-xs">
+          <thead><tr className="text-left text-muted-foreground"><th>Name</th><th>Queries</th><th>Types</th><th>Answers</th><th>Clients</th></tr></thead>
+          <tbody>
+            {((dns.names as Rec[]) || []).map((n, i) => (
+              <tr key={i} className="border-t">
+                <td className="font-mono">{String(n.name)}<DeviceBadge on={n.device} /></td><td>{String(n.queries)}</td>
+                <td>{list(n.types, 4)}</td><td className="font-mono">{list(n.answers, 3)}</td><td>{list(n.clients, 3)}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Section>
+
+      {(Number(http.requests ?? 0) > 0 || (Array.isArray(http.file_transfers) && http.file_transfers.length > 0)) && (
+        <Section title="HTTP" count={Number(http.requests ?? 0)}>
+          <table className="w-full text-xs">
+            <thead><tr className="text-left text-muted-foreground"><th>Host</th><th>Requests</th><th>Methods</th><th>URIs</th><th>Clients</th><th>User agents</th></tr></thead>
+            <tbody>
+              {((http.hosts as Rec[]) || []).map((h, i) => (
+                <tr key={i} className="border-t">
+                  <td className="font-mono">{String(h.host)}<DeviceBadge on={h.device} /></td><td>{String(h.requests)}</td>
+                  <td>{list(h.methods, 4)}</td><td className="font-mono break-all">{list(h.uris, 5)}</td>
+                  <td>{list(h.clients, 3)}</td><td className="break-all">{list(h.user_agents, 2)}</td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+          {Array.isArray(http.file_transfers) && http.file_transfers.length > 0 && (
+            <ul className="mt-2 list-disc pl-5 text-xs">
+              {(http.file_transfers as Rec[]).map((f, i) => (
+                <li key={i}>File transfer: <code>{String(f.filename ?? f.uri ?? f.content_type)}</code> ({String(f.protocol)}, {String(f.direction)}, {String(f.src_ip)} → {String(f.dst_ip)}{f.size ? `, ${kb(f.size)}` : ""})</li>
+              ))}
+            </ul>
+          )}
+        </Section>
+      )}
+
+      <Section title="Clear-text protocols, service events and DHCP">
+        {Array.isArray(clear.flows) && clear.flows.length > 0 ? (
+          <ul className="list-disc pl-5 text-xs">
+            {(clear.flows as Rec[]).map((f, i) => (
+              <li key={i}><span className="font-medium">{String(f.protocol)}</span><DeviceBadge on={f.device} /> · {String(f.flows)} flows, {kb(f.bytes)} · {list(f.pairs, 6)}</li>
+            ))}
+          </ul>
+        ) : <p className="text-xs text-muted-foreground">No clear-text application flows were observed.</p>}
+        {Array.isArray(clear.service_events) && clear.service_events.length > 0 && (
+          <ul className="mt-2 list-disc pl-5 text-xs">
+            {(clear.service_events as Rec[]).map((s, i) => (
+              <li key={i}>{String(s.protocol)} {String(s.kind)}: {String(s.value ?? "")} {String(s.detail ?? "")} ({String(s.src_ip)} → {String(s.dst_ip)})</li>
+            ))}
+          </ul>
+        )}
+        {Array.isArray(clear.dhcp) && clear.dhcp.length > 0 && (
+          <ul className="mt-2 list-disc pl-5 text-xs">
+            {(clear.dhcp as Rec[]).map((d, i) => (
+              <li key={i}>DHCP {String(d.type)}: <span className="font-mono">{String(d.client_mac)}</span> {String(d.hostname ?? "")} {String(d.vendor_class ?? "")} → {String(d.client_ip ?? "")} (server {String(d.server ?? "—")})</li>
+            ))}
+          </ul>
+        )}
+      </Section>
+
+      {protos && (
+        <Section title="Protocols (packets)">
+          <div className="text-xs">{Object.entries(protos).slice(0, 40).map(([k, v]) => `${k} ${v}`).join(" · ")}</div>
+        </Section>
+      )}
+    </div>
+  )
+}

frontend/src/components/providers.tsx

@@ -1,8 +1,10 @@
-'use client'
+"use client"
 
-import { AuthProvider } from '@/lib/auth-context'
+import { useEffect } from "react"
+import { AuthProvider } from "@/lib/auth-context"
+import { installGlobalErrorLogging } from "@/lib/logger"
 
 export function Providers({ children }: { children: React.ReactNode }) {
+  useEffect(() => { installGlobalErrorLogging() }, [])
   return <AuthProvider>{children}</AuthProvider>
 }
-

frontend/src/config/api.ts

@@ -1,10 +1,36 @@
 import axios from "axios"
+import { getLogger } from "@/lib/logger"
 
 export const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || "http://localhost:8000/api"
 
 // The Django API uses an authenticated session shared across localhost ports.
 axios.defaults.withCredentials = true
 
+// Log every axios call with duration, status and the backend request id.
+const log = getLogger("axios")
+axios.interceptors.request.use((config) => {
+  (config as typeof config & { metadata?: { started: number } }).metadata = { started: performance.now() }
+  return config
+})
+axios.interceptors.response.use(
+  (response) => {
+    const started = (response.config as typeof response.config & { metadata?: { started: number } }).metadata?.started
+    const ms = started ? Math.round(performance.now() - started) : -1
+    log.debug(`${(response.config.method || "get").toUpperCase()} ${response.config.url} -> ${response.status} in ${ms} ms`,
+      { requestId: response.headers?.["x-request-id"] })
+    return response
+  },
+  (error) => {
+    const cfg = error?.config || {}
+    const status = error?.response?.status
+    const data = error?.response?.data
+    const message = (data && typeof data === "object" && (data.error || data.detail)) || error?.message
+    log.error(`${(cfg.method || "get").toUpperCase()} ${cfg.url} failed${status ? ` with ${status}` : ""}: ${message}`,
+      { requestId: error?.response?.headers?.["x-request-id"], data })
+    return Promise.reject(error)
+  },
+)
+
 export const getApiUrl = (endpoint: string): string => {
   const path = endpoint.replace(/^\/+/, "")
   return `${API_BASE_URL.replace(/\/+$/, "")}/${path}`

frontend/src/lib/api.ts

@@ -0,0 +1,73 @@
+// fetch wrapper for the Django API: logs every call and turns HTTP errors
+// into ApiError with the server's message and request id.
+
+import { getApiUrl } from "@/config/api"
+import { getLogger, describeError } from "@/lib/logger"
+
+const log = getLogger("api")
+
+export class ApiError extends Error {
+  status: number
+  requestId: string | null
+  body: unknown
+
+  constructor(message: string, status: number, requestId: string | null, body: unknown) {
+    super(message)
+    this.name = "ApiError"
+    this.status = status
+    this.requestId = requestId
+    this.body = body
+  }
+}
+
+// Message shown to the user for a failed call.
+export function apiErrorMessage(err: unknown): string {
+  if (err instanceof ApiError) {
+    const rid = err.requestId ? ` (request ${err.requestId})` : ""
+    return `${err.message}${rid}`
+  }
+  if (err instanceof TypeError) {
+    return "The backend could not be reached. Is the API running and NEXT_PUBLIC_API_URL correct?"
+  }
+  return describeError(err)
+}
+
+async function parseBody(res: Response): Promise<unknown> {
+  const text = await res.text()
+  if (!text) return null
+  try { return JSON.parse(text) } catch { return text }
+}
+
+function messageFromBody(body: unknown, status: number): string {
+  if (body && typeof body === "object") {
+    const b = body as Record<string, unknown>
+    for (const key of ["error", "detail", "message"]) {
+      if (typeof b[key] === "string") return b[key] as string
+    }
+  }
+  if (typeof body === "string" && body.length < 200) return body
+  return `HTTP ${status}`
+}
+
+export async function apiFetch<T = unknown>(endpoint: string, init: RequestInit = {}): Promise<T> {
+  const url = getApiUrl(endpoint)
+  const method = (init.method || "GET").toUpperCase()
+  const started = performance.now()
+  let res: Response
+  try {
+    res = await fetch(url, { cache: "no-store", credentials: "include", ...init })
+  } catch (err) {
+    log.error(`${method} ${endpoint} network failure after ${Math.round(performance.now() - started)} ms: ${describeError(err)}`)
+    throw err
+  }
+  const requestId = res.headers.get("X-Request-ID")
+  const body = await parseBody(res)
+  const ms = Math.round(performance.now() - started)
+  if (!res.ok) {
+    const message = messageFromBody(body, res.status)
+    log.warn(`${method} ${endpoint} -> ${res.status} in ${ms} ms: ${message}`, { requestId, body })
+    throw new ApiError(message, res.status, requestId, body)
+  }
+  log.debug(`${method} ${endpoint} -> ${res.status} in ${ms} ms`, { requestId })
+  return body as T
+}

frontend/src/lib/logger.ts

@@ -0,0 +1,92 @@
+// Browser-side logger with levels, timestamps and a ring buffer.
+//
+// Level: NEXT_PUBLIC_LOG_LEVEL at build time, overridable per browser with
+// localStorage.setItem("iotp_log_level", "debug"). The last 300 entries are
+// kept in window.__iotpLogs and can be copied with window.__iotpDumpLogs().
+
+export type LogLevel = "debug" | "info" | "warn" | "error"
+
+const ORDER: Record<LogLevel, number> = { debug: 10, info: 20, warn: 30, error: 40 }
+const BUFFER_SIZE = 300
+
+interface LogEntry {
+  ts: string
+  level: LogLevel
+  scope: string
+  message: string
+  data?: unknown
+}
+
+declare global {
+  interface Window {
+    __iotpLogs?: LogEntry[]
+    __iotpDumpLogs?: () => string
+  }
+}
+
+function configuredLevel(): LogLevel {
+  let level = (process.env.NEXT_PUBLIC_LOG_LEVEL || (process.env.NODE_ENV === "production" ? "info" : "debug")) as LogLevel
+  if (typeof window !== "undefined") {
+    try {
+      const override = window.localStorage.getItem("iotp_log_level") as LogLevel | null
+      if (override && override in ORDER) level = override
+    } catch { /* storage may be unavailable */ }
+  }
+  return level in ORDER ? level : "info"
+}
+
+function buffer(): LogEntry[] {
+  if (typeof window === "undefined") return []
+  if (!window.__iotpLogs) {
+    window.__iotpLogs = []
+    window.__iotpDumpLogs = () =>
+      (window.__iotpLogs || [])
+        .map((e) => `${e.ts} ${e.level.toUpperCase().padEnd(5)} [${e.scope}] ${e.message}${e.data !== undefined ? " " + safeJson(e.data) : ""}`)
+        .join("\n")
+  }
+  return window.__iotpLogs
+}
+
+function safeJson(v: unknown): string {
+  try { return JSON.stringify(v) } catch { return String(v) }
+}
+
+function emit(level: LogLevel, scope: string, message: string, data?: unknown) {
+  if (ORDER[level] < ORDER[configuredLevel()]) return
+  const ts = new Date().toISOString()
+  const buf = buffer()
+  buf.push({ ts, level, scope, message, data })
+  if (buf.length > BUFFER_SIZE) buf.splice(0, buf.length - BUFFER_SIZE)
+  const line = `[iotp ${ts.slice(11, 23)}] [${scope}] ${message}`
+  const fn = level === "error" ? console.error : level === "warn" ? console.warn : level === "info" ? console.info : console.debug
+  if (data !== undefined) fn(line, data)
+  else fn(line)
+}
+
+export function getLogger(scope: string) {
+  return {
+    debug: (message: string, data?: unknown) => emit("debug", scope, message, data),
+    info: (message: string, data?: unknown) => emit("info", scope, message, data),
+    warn: (message: string, data?: unknown) => emit("warn", scope, message, data),
+    error: (message: string, data?: unknown) => emit("error", scope, message, data),
+  }
+}
+
+// Describe any thrown value for logs and user messages.
+export function describeError(err: unknown): string {
+  if (err instanceof Error) return `${err.name}: ${err.message}`
+  if (typeof err === "string") return err
+  return safeJson(err)
+}
+
+// Global handlers so uncaught browser errors land in the buffer too.
+export function installGlobalErrorLogging() {
+  if (typeof window === "undefined") return
+  const w = window as Window & { __iotpGlobalInstalled?: boolean }
+  if (w.__iotpGlobalInstalled) return
+  w.__iotpGlobalInstalled = true
+  const log = getLogger("window")
+  window.addEventListener("error", (ev) => log.error(`Uncaught: ${ev.message}`, { file: ev.filename, line: ev.lineno }))
+  window.addEventListener("unhandledrejection", (ev) => log.error(`Unhandled rejection: ${describeError(ev.reason)}`))
+  log.debug(`Logging active at level "${configuredLevel()}"; call window.__iotpDumpLogs() to copy the buffer`)
+}

requirements.txt

@@ -5,5 +5,5 @@ django-cors-headers==4.6.0
 djangorestframework==3.15.2
 dnspython==2.7.0
 gunicorn==23.0.0
-pyshark==0.6
 python-nmap==0.7.1
+cryptography>=42.0