"""Build a complete observation inventory from the capture context. Findings say what is wrong; the inventory says what was there. """ from __future__ import annotations from collections import defaultdict from typing import Optional from .context import CaptureContext, Host, is_special _MAX_HOSTS = 200 _MAX_ENDPOINTS = 400 _MAX_DNS_NAMES = 500 _MAX_TLS = 400 # Well-known service names for ports without a dissected protocol. _PORT_NAMES = { 20: "ftp-data", 21: "ftp", 22: "ssh", 23: "telnet", 25: "smtp", 53: "dns", 67: "dhcp", 68: "dhcp", 69: "tftp", 80: "http", 110: "pop3", 123: "ntp", 137: "netbios-ns", 138: "netbios-dgm", 139: "netbios-ssn", 143: "imap", 161: "snmp", 162: "snmp-trap", 389: "ldap", 443: "https", 445: "smb", 465: "smtps", 500: "ike", 502: "modbus", 514: "syslog", 554: "rtsp", 587: "submission", 631: "ipp", 636: "ldaps", 853: "dot", 873: "rsync", 993: "imaps", 995: "pop3s", 1080: "socks", 1194: "openvpn", 1433: "mssql", 1521: "oracle", 1883: "mqtt", 1900: "ssdp", 2049: "nfs", 3306: "mysql", 3389: "rdp", 4500: "ipsec-nat-t", 5000: "upnp/http-alt", 5060: "sip", 5061: "sips", 5222: "xmpp", 5223: "apns", 5228: "gcm", 5353: "mdns", 5355: "llmnr", 5432: "postgresql", 5683: "coap", 5684: "coaps", 5900: "vnc", 6379: "redis", 6667: "irc", 8000: "http-alt", 8080: "http-alt", 8443: "https-alt", 8883: "mqtts", 8888: "http-alt", 9000: "http-alt", 9100: "jetdirect", 11211: "memcached", 27017: "mongodb", 49152: "upnp", } # Dissector helper names that never describe a service by themselves. _NOISE_PROTOS = {"x509ce", "x509sat", "x509af", "x509if", "pkix1implicit", "pkix1explicit", "pkcs-1", "ber", "tcp.segments", "data", "data-text-lines", "mime_multipart", "image-jfif", "image-gif", "png", "json", "xml", "urlencoded-form", "media"} def _clean(protos) -> list[str]: return sorted(p for p in protos if p not in _NOISE_PROTOS) _CLEARTEXT_APP = {"http", "ftp", "telnet", "pop", "imap", "smtp", "snmp", "tftp", "mqtt", "coap", "rtsp", "sip", "ldap", "mbtcp", "modbus", "nbns", "smb", "smb2", "syslog", "irc", "xmpp", "vnc", "rdp"} def build_inventory(ctx: CaptureContext) -> dict: inv: dict = {} inv["capture"] = _capture_section(ctx) inv["device"] = _device_section(ctx) inv["hosts"] = _hosts_section(ctx) inv["external_endpoints"] = _endpoints_section(ctx) inv["protocols"] = _protocols_section(ctx) inv["services"] = _services_section(ctx) inv["tls"] = _tls_section(ctx) inv["dns"] = _dns_section(ctx) inv["http"] = _http_section(ctx) inv["cleartext"] = _cleartext_section(ctx) inv["coverage"] = _coverage_section(ctx) return inv # --------------------------------------------------------------------------- # def _capture_section(ctx: CaptureContext) -> dict: return { "packets": ctx.packet_count, "duration_seconds": round(ctx.duration, 3), "start": ctx.start_time, "end": ctx.end_time, "vlans": sorted(ctx.vlan_ids), "hosts_total": len([h for h in ctx.hosts.values() if not is_special(h.ip)]), "internal_hosts": len([h for h in ctx.hosts.values() if h.is_private and not is_special(h.ip)]), "external_hosts": len([h for h in ctx.hosts.values() if not h.is_private]), "conversations": len(ctx.conversations), "extractor": ctx.extractor, } def _host_dict(ctx: CaptureContext, h: Host) -> dict: ext = {c.dst_ip for c in ctx.conversations if c.src_ip == h.ip and ctx.is_external(c.dst_ip)} ext |= {c.src_ip for c in ctx.conversations if c.dst_ip == h.ip and ctx.is_external(c.src_ip)} return { "ip": h.ip, "macs": sorted(h.macs), "names": sorted(h.hostnames), "dhcp_vendor_class": h.dhcp_vendor_class, "is_device": ctx.is_device(h.ip), "is_private": h.is_private, "packets": h.packets, "bytes_sent": h.bytes_sent, "bytes_received": h.bytes_received, "first_seen": h.first_seen, "last_seen": h.last_seen, "ports_offered": sorted(p for p in h.ports_offered if p is not None)[:60], "ports_contacted": sorted(p for p in h.ports_contacted if p is not None)[:60], "protocols": _clean(h.app_protocols)[:40], "external_endpoints": len(ext), "user_agents": sorted(h.user_agents)[:10], } def _device_section(ctx: CaptureContext) -> Optional[dict]: if not ctx.device_ip: # Suggest likely devices so the operator can re-run with --device-ip. cands = [] for h in ctx.hosts.values(): if not h.is_private or is_special(h.ip): continue score = h.packets if h.hostnames: score *= 2 cands.append((score, h)) cands.sort(key=lambda t: -t[0]) return { "configured": False, "candidates": [ {"ip": h.ip, "names": sorted(h.hostnames), "macs": sorted(h.macs), "packets": h.packets} for _, h in cands[:5] ], } h = ctx.hosts.get(ctx.device_ip) if h is None: return {"configured": True, "ip": ctx.device_ip, "seen": False, "note": "The configured device address does not appear in the capture."} d = _host_dict(ctx, h) d.update({"configured": True, "seen": True}) # Everything the device talked to. peers: dict[str, dict] = {} for c in ctx.conversations: if c.src_ip == ctx.device_ip: peer, port = c.dst_ip, c.dst_port elif c.dst_ip == ctx.device_ip: peer, port = c.src_ip, c.src_port else: continue e = peers.setdefault(peer, {"ip": peer, "external": ctx.is_external(peer), "ports": set(), "flows": 0, "bytes_to_peer": 0, "bytes_from_peer": 0, "protocols": set(), "names": set()}) e["flows"] += 1 if port: e["ports"].add(port) if c.src_ip == ctx.device_ip: e["bytes_to_peer"] += c.bytes_src_to_dst e["bytes_from_peer"] += c.bytes_dst_to_src else: e["bytes_to_peer"] += c.bytes_dst_to_src e["bytes_from_peer"] += c.bytes_src_to_dst e["protocols"] |= c.app_protocols for peer, e in peers.items(): e["names"] = sorted(ctx.names_for_ip(peer))[:8] e["ports"] = sorted(e["ports"])[:20] e["protocols"] = _clean(e["protocols"])[:15] d["peers"] = sorted(peers.values(), key=lambda e: -(e["bytes_to_peer"] + e["bytes_from_peer"]))[:100] d["dns_queries"] = sorted({q.query_name.rstrip(".").lower() for q in ctx.dns_queries if q.src_ip == ctx.device_ip and not q.is_response and q.query_name})[:100] d["tls_sessions"] = len([t for t in ctx.tls_sessions if t.src_ip == ctx.device_ip]) d["cleartext_flows"] = len([c for c in ctx.conversations if ctx.device_ip in (c.src_ip, c.dst_ip) and c.payload_bytes > 0 and not c.is_encrypted]) return d def _hosts_section(ctx: CaptureContext) -> list[dict]: hosts = [h for h in ctx.hosts.values() if not is_special(h.ip) and h.is_private] hosts.sort(key=lambda h: (-int(ctx.is_device(h.ip)), -h.packets)) return [_host_dict(ctx, h) for h in hosts[:_MAX_HOSTS]] def _endpoints_section(ctx: CaptureContext) -> list[dict]: eps: dict[str, dict] = {} for c in ctx.conversations: if ctx.is_external(c.dst_ip) and not ctx.is_external(c.src_ip): ext, internal, port = c.dst_ip, c.src_ip, c.dst_port up, down = c.bytes_src_to_dst, c.bytes_dst_to_src elif ctx.is_external(c.src_ip) and not ctx.is_external(c.dst_ip): ext, internal, port = c.src_ip, c.dst_ip, c.src_port up, down = c.bytes_dst_to_src, c.bytes_src_to_dst else: continue e = eps.setdefault(ext, {"ip": ext, "ports": set(), "protocols": set(), "flows": 0, "bytes_out": 0, "bytes_in": 0, "internal_hosts": set(), "first_seen": c.first_seen, "last_seen": c.last_seen, "device": False}) e["flows"] += 1 e["bytes_out"] += up e["bytes_in"] += down if port: e["ports"].add(port) e["protocols"] |= c.app_protocols e["internal_hosts"].add(internal) if ctx.is_device(internal): e["device"] = True if c.first_seen is not None: e["first_seen"] = c.first_seen if e["first_seen"] is None else min(e["first_seen"], c.first_seen) if c.last_seen is not None: e["last_seen"] = c.last_seen if e["last_seen"] is None else max(e["last_seen"], c.last_seen) out = [] for ip, e in eps.items(): e["names"] = sorted(ctx.names_for_ip(ip))[:8] e["ports"] = sorted(e["ports"])[:20] e["protocols"] = _clean(e["protocols"])[:15] e["internal_hosts"] = sorted(e["internal_hosts"])[:10] e["encrypted"] = bool(set(e["protocols"]) & {"tls", "quic", "ssh", "dtls"}) out.append(e) out.sort(key=lambda e: (-int(e["device"]), -(e["bytes_out"] + e["bytes_in"]))) return out[:_MAX_ENDPOINTS] def _protocols_section(ctx: CaptureContext) -> dict: skip = {"eth", "ethertype", "vlan", "frame", "sll", "llc", "tcp.segments", "data", "data-text-lines", "x509ce", "x509sat", "x509af", "x509if", "pkix1implicit", "pkix1explicit", "pkcs-1", "ber", "mime_multipart"} app = {k: v for k, v in ctx.app_protocol_stats.items() if k not in skip} return { "by_protocol": dict(sorted(app.items(), key=lambda kv: -kv[1])), "highest_layer": dict(sorted(ctx.protocol_stats.items(), key=lambda kv: -kv[1])[:30]), "icmp": dict(ctx.icmp_stats), } def _services_section(ctx: CaptureContext) -> list[dict]: """Server-side services observed: who answered on which port with which protocol.""" svc: dict[tuple, dict] = {} for c in ctx.conversations: if not c.answered or not c.dst_port: continue key = (c.dst_ip, c.dst_port, c.protocol) e = svc.setdefault(key, {"ip": c.dst_ip, "port": c.dst_port, "transport": c.protocol, "protocols": set(), "clients": set(), "flows": 0, "external": ctx.is_external(c.dst_ip), "device": ctx.is_device(c.dst_ip)}) e["flows"] += 1 e["clients"].add(c.src_ip) e["protocols"] |= c.app_protocols out = [] for e in svc.values(): protos = _clean(e["protocols"]) e["protocols"] = protos e["service"] = _service_name(e["port"], protos) e["encrypted"] = bool(set(protos) & {"tls", "quic", "ssh", "dtls"}) e["cleartext_app"] = bool(set(protos) & _CLEARTEXT_APP) and not e["encrypted"] e["clients"] = sorted(e["clients"])[:10] out.append(e) out.sort(key=lambda e: (-int(e["device"]), int(e["external"]), e["ip"], e["port"])) return out[:_MAX_ENDPOINTS] def _service_name(port: int, protos: list[str]) -> str: for p in ("mqtt", "coap", "http", "rtsp", "ssh", "telnet", "ftp", "smtp", "pop", "imap", "snmp", "dns", "mdns", "ssdp", "dhcp", "ntp", "tftp", "sip", "smb2", "smb", "ldap", "mbtcp", "vnc", "rdp", "xmpp", "quic", "tls"): if p in protos: base = {"pop": "pop3", "mbtcp": "modbus"}.get(p, p) if base == "tls": return _PORT_NAMES.get(port, f"tls/{port}") return base return _PORT_NAMES.get(port, f"port {port}") def _tls_section(ctx: CaptureContext) -> dict: sessions = [] for t in ctx.tls_sessions: sessions.append({ "client": t.src_ip, "server": t.dst_ip, "port": t.dst_port, "transport": t.transport, "sni": t.sni, "version": t.version, "cipher": t.cipher, "client_max_version": t.client_version, "client_offered_ciphers": len(t.client_offered_ciphers), "alpn": t.alpn[:4], "handshake_complete": t.server_hello_seen, "certificate": { "subject": t.cert_subject_cn, "issuer": t.cert_issuer_cn, "san": t.cert_san[:5], "not_after": t.cert_not_after, "self_signed": t.cert_self_signed, "expired": t.cert_expired, "signature": t.cert_sig_algorithm, "key": f"{t.cert_key_type} {t.cert_key_bits}" if t.cert_key_type else None, } if t.cert_seen else None, "mutual_tls": t.client_cert_seen, "alerts": t.alerts, "device": ctx.is_device(t.src_ip) or ctx.is_device(t.dst_ip), "packet": t.packet_number, }) sessions.sort(key=lambda s: (-int(s["device"]), s["client"] or "", s["server"] or "")) versions: dict[str, int] = defaultdict(int) ciphers: dict[str, int] = defaultdict(int) for t in ctx.tls_sessions: if t.version: versions[t.version] += 1 if t.cipher: ciphers[t.cipher] += 1 return { "sessions_total": len(ctx.tls_sessions), "versions": dict(versions), "ciphers": dict(ciphers), "sessions": sessions[:_MAX_TLS], } def _dns_section(ctx: CaptureContext) -> dict: names: dict[str, dict] = {} rcodes: dict[str, int] = defaultdict(int) qtypes: dict[str, int] = defaultdict(int) resolvers: dict[str, int] = defaultdict(int) for q in ctx.dns_queries: name = (q.query_name or "").rstrip(".").lower() if not name: continue e = names.setdefault(name, {"name": name, "queries": 0, "clients": set(), "answers": set(), "rcodes": set(), "types": set(), "transport": q.transport}) if q.is_response: if q.rcode: rcodes[q.rcode] += 1 e["rcodes"].add(q.rcode) e["answers"].update(q.response_ips[:6]) if q.resolver: resolvers[q.resolver] += 1 else: e["queries"] += 1 if q.src_ip: e["clients"].add(q.src_ip) if q.qtype: qtypes[q.qtype] += 1 e["types"].add(q.qtype) out = [] for e in names.values(): e["clients"] = sorted(e["clients"])[:6] e["answers"] = sorted(e["answers"])[:6] e["rcodes"] = sorted(e["rcodes"]) e["types"] = sorted(e["types"]) e["device"] = ctx.device_ip in e["clients"] if ctx.device_ip else False out.append(e) out.sort(key=lambda e: (-int(e["device"]), -e["queries"], e["name"])) return { "unique_names": len(names), "queries": sum(1 for q in ctx.dns_queries if not q.is_response), "responses": sum(1 for q in ctx.dns_queries if q.is_response), "rcodes": dict(rcodes), "query_types": dict(qtypes), "resolvers": dict(resolvers), "names": out[:_MAX_DNS_NAMES], } def _http_section(ctx: CaptureContext) -> dict: hosts: dict[str, dict] = {} for r in ctx.http_requests: key = str(r.host or r.dst_ip or "?") e = hosts.setdefault(key, {"host": key, "server_ip": r.dst_ip, "port": r.dst_port, "requests": 0, "methods": set(), "uris": [], "clients": set(), "user_agents": set(), "auth": False}) e["requests"] += 1 if r.method: e["methods"].add(r.method) if r.uri and len(e["uris"]) < 15 and r.uri not in e["uris"]: e["uris"].append(r.uri) if r.src_ip: e["clients"].add(r.src_ip) if r.user_agent: e["user_agents"].add(r.user_agent[:100]) if r.authorization: e["auth"] = True out = [] for e in hosts.values(): e["methods"] = sorted(e["methods"]) e["clients"] = sorted(e["clients"])[:6] e["user_agents"] = sorted(e["user_agents"])[:5] e["device"] = bool(ctx.device_ip and (ctx.device_ip in e["clients"] or e["server_ip"] == ctx.device_ip)) out.append(e) out.sort(key=lambda e: (-int(e["device"]), -e["requests"])) status: dict[str, int] = defaultdict(int) for resp in ctx.http_responses: if resp.status_code is not None: status[str(resp.status_code)] += 1 return {"requests": len(ctx.http_requests), "responses": len(ctx.http_responses), "status_codes": dict(status), "hosts": out[:200], "file_transfers": [ {"filename": f.filename, "extension": f.extension, "content_type": f.content_type, "size": f.size, "protocol": f.protocol, "direction": f.direction, "src_ip": f.src_ip, "dst_ip": f.dst_ip, "uri": f.uri, "packet": f.packet_number} for f in ctx.file_transfers[:100] ]} def _cleartext_section(ctx: CaptureContext) -> dict: """Unencrypted application flows and clear-text services grouped by protocol.""" flows: dict[str, dict] = {} for c in ctx.conversations: if c.payload_bytes <= 0 or c.is_encrypted: continue protos = sorted(c.app_protocols & _CLEARTEXT_APP) or ([_PORT_NAMES[c.dst_port]] if c.dst_port in _PORT_NAMES and c.dst_port not in (53, 123, 67, 68, 5353, 1900, 137, 138) else []) if not protos: continue for p in protos: e = flows.setdefault(p, {"protocol": p, "flows": 0, "bytes": 0, "pairs": set(), "device": False}) e["flows"] += 1 e["bytes"] += c.payload_bytes e["pairs"].add(f"{c.src_ip} -> {c.dst_ip}:{c.dst_port}") if ctx.is_device(c.src_ip) or ctx.is_device(c.dst_ip): e["device"] = True out = [] for e in flows.values(): e["pairs"] = sorted(e["pairs"])[:20] out.append(e) out.sort(key=lambda e: (-int(e["device"]), -e["bytes"])) services = [ {"protocol": s.protocol, "kind": s.kind, "value": s.value, "detail": s.detail, "src_ip": s.src_ip, "dst_ip": s.dst_ip, "port": s.dst_port, "packet": s.packet_number} for s in ctx.service_events[:200] ] return {"flows": out, "service_events": services, "mqtt_events": len(ctx.mqtt_events), "dhcp": [ {"type": d.message_type, "client_mac": d.client_mac, "client_ip": d.client_ip, "hostname": d.hostname, "vendor_class": d.vendor_class, "server": d.server_id} for d in ctx.dhcp_events[:60] ]} def _coverage_section(ctx: CaptureContext) -> dict: notes = list(ctx.coverage_notes) if ctx.truncated: notes.insert(0, ctx.truncation_reason or "Extraction was truncated.") encrypted_flows = sum(1 for c in ctx.conversations if c.is_encrypted) payload_flows = sum(1 for c in ctx.conversations if c.payload_bytes > 0) undecoded = [c for c in ctx.conversations if c.payload_bytes > 0 and not c.app_protocols] undecoded_ports: dict[int, int] = defaultdict(int) for c in undecoded: if c.dst_port: undecoded_ports[c.dst_port] += 1 if not ctx.device_ip: notes.append("No device address was configured; findings are not attributed to a device under test.") tls13 = sum(1 for t in ctx.tls_sessions if t.version == "TLS 1.3") if tls13: notes.append(f"{tls13} TLS 1.3 sessions: server certificates are encrypted and could not be inspected.") return { "complete": not ctx.truncated, "notes": notes, "flows_total": len(ctx.conversations), "flows_with_payload": payload_flows, "flows_encrypted": encrypted_flows, "flows_undecoded": len(undecoded), "undecoded_ports": dict(sorted(undecoded_ports.items(), key=lambda kv: -kv[1])[:15]), "text_payloads_scanned": len(ctx.text_payloads), }