Files

439 lines
20 KiB
Python

"""Build a complete observation inventory from the capture context.
Findings say what is wrong; the inventory says what was there.
"""
from __future__ import annotations
from collections import defaultdict
from typing import Optional
from .context import CaptureContext, Host, is_special
_MAX_HOSTS = 200
_MAX_ENDPOINTS = 400
_MAX_DNS_NAMES = 500
_MAX_TLS = 400
# Well-known service names for ports without a dissected protocol.
_PORT_NAMES = {
20: "ftp-data", 21: "ftp", 22: "ssh", 23: "telnet", 25: "smtp", 53: "dns", 67: "dhcp",
68: "dhcp", 69: "tftp", 80: "http", 110: "pop3", 123: "ntp", 137: "netbios-ns",
138: "netbios-dgm", 139: "netbios-ssn", 143: "imap", 161: "snmp", 162: "snmp-trap",
389: "ldap", 443: "https", 445: "smb", 465: "smtps", 500: "ike", 502: "modbus",
514: "syslog", 554: "rtsp", 587: "submission", 631: "ipp", 636: "ldaps", 853: "dot",
873: "rsync", 993: "imaps", 995: "pop3s", 1080: "socks", 1194: "openvpn",
1433: "mssql", 1521: "oracle", 1883: "mqtt", 1900: "ssdp", 2049: "nfs",
3306: "mysql", 3389: "rdp", 4500: "ipsec-nat-t", 5000: "upnp/http-alt", 5060: "sip",
5061: "sips", 5222: "xmpp", 5223: "apns", 5228: "gcm", 5353: "mdns", 5355: "llmnr",
5432: "postgresql", 5683: "coap", 5684: "coaps", 5900: "vnc", 6379: "redis",
6667: "irc", 8000: "http-alt", 8080: "http-alt", 8443: "https-alt", 8883: "mqtts",
8888: "http-alt", 9000: "http-alt", 9100: "jetdirect", 11211: "memcached",
27017: "mongodb", 49152: "upnp",
}
# Dissector helper names that never describe a service by themselves.
_NOISE_PROTOS = {"x509ce", "x509sat", "x509af", "x509if", "pkix1implicit", "pkix1explicit",
"pkcs-1", "ber", "tcp.segments", "data", "data-text-lines", "mime_multipart",
"image-jfif", "image-gif", "png", "json", "xml", "urlencoded-form", "media"}
def _clean(protos) -> list[str]:
return sorted(p for p in protos if p not in _NOISE_PROTOS)
_CLEARTEXT_APP = {"http", "ftp", "telnet", "pop", "imap", "smtp", "snmp", "tftp", "mqtt",
"coap", "rtsp", "sip", "ldap", "mbtcp", "modbus", "nbns", "smb", "smb2",
"syslog", "irc", "xmpp", "vnc", "rdp"}
def build_inventory(ctx: CaptureContext) -> dict:
inv: dict = {}
inv["capture"] = _capture_section(ctx)
inv["device"] = _device_section(ctx)
inv["hosts"] = _hosts_section(ctx)
inv["external_endpoints"] = _endpoints_section(ctx)
inv["protocols"] = _protocols_section(ctx)
inv["services"] = _services_section(ctx)
inv["tls"] = _tls_section(ctx)
inv["dns"] = _dns_section(ctx)
inv["http"] = _http_section(ctx)
inv["cleartext"] = _cleartext_section(ctx)
inv["coverage"] = _coverage_section(ctx)
return inv
# --------------------------------------------------------------------------- #
def _capture_section(ctx: CaptureContext) -> dict:
return {
"packets": ctx.packet_count,
"duration_seconds": round(ctx.duration, 3),
"start": ctx.start_time, "end": ctx.end_time,
"vlans": sorted(ctx.vlan_ids),
"hosts_total": len([h for h in ctx.hosts.values() if not is_special(h.ip)]),
"internal_hosts": len([h for h in ctx.hosts.values() if h.is_private and not is_special(h.ip)]),
"external_hosts": len([h for h in ctx.hosts.values() if not h.is_private]),
"conversations": len(ctx.conversations),
"extractor": ctx.extractor,
}
def _host_dict(ctx: CaptureContext, h: Host) -> dict:
ext = {c.dst_ip for c in ctx.conversations if c.src_ip == h.ip and ctx.is_external(c.dst_ip)}
ext |= {c.src_ip for c in ctx.conversations if c.dst_ip == h.ip and ctx.is_external(c.src_ip)}
return {
"ip": h.ip,
"macs": sorted(h.macs),
"names": sorted(h.hostnames),
"dhcp_vendor_class": h.dhcp_vendor_class,
"is_device": ctx.is_device(h.ip),
"is_private": h.is_private,
"packets": h.packets,
"bytes_sent": h.bytes_sent, "bytes_received": h.bytes_received,
"first_seen": h.first_seen, "last_seen": h.last_seen,
"ports_offered": sorted(p for p in h.ports_offered if p is not None)[:60],
"ports_contacted": sorted(p for p in h.ports_contacted if p is not None)[:60],
"protocols": _clean(h.app_protocols)[:40],
"external_endpoints": len(ext),
"user_agents": sorted(h.user_agents)[:10],
}
def _device_section(ctx: CaptureContext) -> Optional[dict]:
if not ctx.device_ip:
# Suggest likely devices so the operator can re-run with --device-ip.
cands = []
for h in ctx.hosts.values():
if not h.is_private or is_special(h.ip):
continue
score = h.packets
if h.hostnames:
score *= 2
cands.append((score, h))
cands.sort(key=lambda t: -t[0])
return {
"configured": False,
"candidates": [
{"ip": h.ip, "names": sorted(h.hostnames), "macs": sorted(h.macs), "packets": h.packets}
for _, h in cands[:5]
],
}
h = ctx.hosts.get(ctx.device_ip)
if h is None:
return {"configured": True, "ip": ctx.device_ip, "seen": False,
"note": "The configured device address does not appear in the capture."}
d = _host_dict(ctx, h)
d.update({"configured": True, "seen": True})
# Everything the device talked to.
peers: dict[str, dict] = {}
for c in ctx.conversations:
if c.src_ip == ctx.device_ip:
peer, port = c.dst_ip, c.dst_port
elif c.dst_ip == ctx.device_ip:
peer, port = c.src_ip, c.src_port
else:
continue
e = peers.setdefault(peer, {"ip": peer, "external": ctx.is_external(peer), "ports": set(),
"flows": 0, "bytes_to_peer": 0, "bytes_from_peer": 0,
"protocols": set(), "names": set()})
e["flows"] += 1
if port:
e["ports"].add(port)
if c.src_ip == ctx.device_ip:
e["bytes_to_peer"] += c.bytes_src_to_dst
e["bytes_from_peer"] += c.bytes_dst_to_src
else:
e["bytes_to_peer"] += c.bytes_dst_to_src
e["bytes_from_peer"] += c.bytes_src_to_dst
e["protocols"] |= c.app_protocols
for peer, e in peers.items():
e["names"] = sorted(ctx.names_for_ip(peer))[:8]
e["ports"] = sorted(e["ports"])[:20]
e["protocols"] = _clean(e["protocols"])[:15]
d["peers"] = sorted(peers.values(), key=lambda e: -(e["bytes_to_peer"] + e["bytes_from_peer"]))[:100]
d["dns_queries"] = sorted({q.query_name.rstrip(".").lower() for q in ctx.dns_queries
if q.src_ip == ctx.device_ip and not q.is_response and q.query_name})[:100]
d["tls_sessions"] = len([t for t in ctx.tls_sessions if t.src_ip == ctx.device_ip])
d["cleartext_flows"] = len([c for c in ctx.conversations
if ctx.device_ip in (c.src_ip, c.dst_ip)
and c.payload_bytes > 0 and not c.is_encrypted])
return d
def _hosts_section(ctx: CaptureContext) -> list[dict]:
hosts = [h for h in ctx.hosts.values() if not is_special(h.ip) and h.is_private]
hosts.sort(key=lambda h: (-int(ctx.is_device(h.ip)), -h.packets))
return [_host_dict(ctx, h) for h in hosts[:_MAX_HOSTS]]
def _endpoints_section(ctx: CaptureContext) -> list[dict]:
eps: dict[str, dict] = {}
for c in ctx.conversations:
if ctx.is_external(c.dst_ip) and not ctx.is_external(c.src_ip):
ext, internal, port = c.dst_ip, c.src_ip, c.dst_port
up, down = c.bytes_src_to_dst, c.bytes_dst_to_src
elif ctx.is_external(c.src_ip) and not ctx.is_external(c.dst_ip):
ext, internal, port = c.src_ip, c.dst_ip, c.src_port
up, down = c.bytes_dst_to_src, c.bytes_src_to_dst
else:
continue
e = eps.setdefault(ext, {"ip": ext, "ports": set(), "protocols": set(), "flows": 0,
"bytes_out": 0, "bytes_in": 0, "internal_hosts": set(),
"first_seen": c.first_seen, "last_seen": c.last_seen,
"device": False})
e["flows"] += 1
e["bytes_out"] += up
e["bytes_in"] += down
if port:
e["ports"].add(port)
e["protocols"] |= c.app_protocols
e["internal_hosts"].add(internal)
if ctx.is_device(internal):
e["device"] = True
if c.first_seen is not None:
e["first_seen"] = c.first_seen if e["first_seen"] is None else min(e["first_seen"], c.first_seen)
if c.last_seen is not None:
e["last_seen"] = c.last_seen if e["last_seen"] is None else max(e["last_seen"], c.last_seen)
out = []
for ip, e in eps.items():
e["names"] = sorted(ctx.names_for_ip(ip))[:8]
e["ports"] = sorted(e["ports"])[:20]
e["protocols"] = _clean(e["protocols"])[:15]
e["internal_hosts"] = sorted(e["internal_hosts"])[:10]
e["encrypted"] = bool(set(e["protocols"]) & {"tls", "quic", "ssh", "dtls"})
out.append(e)
out.sort(key=lambda e: (-int(e["device"]), -(e["bytes_out"] + e["bytes_in"])))
return out[:_MAX_ENDPOINTS]
def _protocols_section(ctx: CaptureContext) -> dict:
skip = {"eth", "ethertype", "vlan", "frame", "sll", "llc", "tcp.segments", "data",
"data-text-lines", "x509ce", "x509sat", "x509af", "x509if", "pkix1implicit",
"pkix1explicit", "pkcs-1", "ber", "mime_multipart"}
app = {k: v for k, v in ctx.app_protocol_stats.items() if k not in skip}
return {
"by_protocol": dict(sorted(app.items(), key=lambda kv: -kv[1])),
"highest_layer": dict(sorted(ctx.protocol_stats.items(), key=lambda kv: -kv[1])[:30]),
"icmp": dict(ctx.icmp_stats),
}
def _services_section(ctx: CaptureContext) -> list[dict]:
"""Server-side services observed: who answered on which port with which protocol."""
svc: dict[tuple, dict] = {}
for c in ctx.conversations:
if not c.answered or not c.dst_port:
continue
key = (c.dst_ip, c.dst_port, c.protocol)
e = svc.setdefault(key, {"ip": c.dst_ip, "port": c.dst_port, "transport": c.protocol,
"protocols": set(), "clients": set(), "flows": 0,
"external": ctx.is_external(c.dst_ip), "device": ctx.is_device(c.dst_ip)})
e["flows"] += 1
e["clients"].add(c.src_ip)
e["protocols"] |= c.app_protocols
out = []
for e in svc.values():
protos = _clean(e["protocols"])
e["protocols"] = protos
e["service"] = _service_name(e["port"], protos)
e["encrypted"] = bool(set(protos) & {"tls", "quic", "ssh", "dtls"})
e["cleartext_app"] = bool(set(protos) & _CLEARTEXT_APP) and not e["encrypted"]
e["clients"] = sorted(e["clients"])[:10]
out.append(e)
out.sort(key=lambda e: (-int(e["device"]), int(e["external"]), e["ip"], e["port"]))
return out[:_MAX_ENDPOINTS]
def _service_name(port: int, protos: list[str]) -> str:
for p in ("mqtt", "coap", "http", "rtsp", "ssh", "telnet", "ftp", "smtp", "pop", "imap",
"snmp", "dns", "mdns", "ssdp", "dhcp", "ntp", "tftp", "sip", "smb2", "smb", "ldap",
"mbtcp", "vnc", "rdp", "xmpp", "quic", "tls"):
if p in protos:
base = {"pop": "pop3", "mbtcp": "modbus"}.get(p, p)
if base == "tls":
return _PORT_NAMES.get(port, f"tls/{port}")
return base
return _PORT_NAMES.get(port, f"port {port}")
def _tls_section(ctx: CaptureContext) -> dict:
sessions = []
for t in ctx.tls_sessions:
sessions.append({
"client": t.src_ip, "server": t.dst_ip, "port": t.dst_port, "transport": t.transport,
"sni": t.sni, "version": t.version, "cipher": t.cipher,
"client_max_version": t.client_version,
"client_offered_ciphers": len(t.client_offered_ciphers),
"alpn": t.alpn[:4], "handshake_complete": t.server_hello_seen,
"certificate": {
"subject": t.cert_subject_cn, "issuer": t.cert_issuer_cn, "san": t.cert_san[:5],
"not_after": t.cert_not_after, "self_signed": t.cert_self_signed,
"expired": t.cert_expired, "signature": t.cert_sig_algorithm,
"key": f"{t.cert_key_type} {t.cert_key_bits}" if t.cert_key_type else None,
} if t.cert_seen else None,
"mutual_tls": t.client_cert_seen,
"alerts": t.alerts,
"device": ctx.is_device(t.src_ip) or ctx.is_device(t.dst_ip),
"packet": t.packet_number,
})
sessions.sort(key=lambda s: (-int(s["device"]), s["client"] or "", s["server"] or ""))
versions: dict[str, int] = defaultdict(int)
ciphers: dict[str, int] = defaultdict(int)
for t in ctx.tls_sessions:
if t.version:
versions[t.version] += 1
if t.cipher:
ciphers[t.cipher] += 1
return {
"sessions_total": len(ctx.tls_sessions),
"versions": dict(versions), "ciphers": dict(ciphers),
"sessions": sessions[:_MAX_TLS],
}
def _dns_section(ctx: CaptureContext) -> dict:
names: dict[str, dict] = {}
rcodes: dict[str, int] = defaultdict(int)
qtypes: dict[str, int] = defaultdict(int)
resolvers: dict[str, int] = defaultdict(int)
for q in ctx.dns_queries:
name = (q.query_name or "").rstrip(".").lower()
if not name:
continue
e = names.setdefault(name, {"name": name, "queries": 0, "clients": set(), "answers": set(),
"rcodes": set(), "types": set(), "transport": q.transport})
if q.is_response:
if q.rcode:
rcodes[q.rcode] += 1
e["rcodes"].add(q.rcode)
e["answers"].update(q.response_ips[:6])
if q.resolver:
resolvers[q.resolver] += 1
else:
e["queries"] += 1
if q.src_ip:
e["clients"].add(q.src_ip)
if q.qtype:
qtypes[q.qtype] += 1
e["types"].add(q.qtype)
out = []
for e in names.values():
e["clients"] = sorted(e["clients"])[:6]
e["answers"] = sorted(e["answers"])[:6]
e["rcodes"] = sorted(e["rcodes"])
e["types"] = sorted(e["types"])
e["device"] = ctx.device_ip in e["clients"] if ctx.device_ip else False
out.append(e)
out.sort(key=lambda e: (-int(e["device"]), -e["queries"], e["name"]))
return {
"unique_names": len(names),
"queries": sum(1 for q in ctx.dns_queries if not q.is_response),
"responses": sum(1 for q in ctx.dns_queries if q.is_response),
"rcodes": dict(rcodes), "query_types": dict(qtypes), "resolvers": dict(resolvers),
"names": out[:_MAX_DNS_NAMES],
}
def _http_section(ctx: CaptureContext) -> dict:
hosts: dict[str, dict] = {}
for r in ctx.http_requests:
key = str(r.host or r.dst_ip or "?")
e = hosts.setdefault(key, {"host": key, "server_ip": r.dst_ip, "port": r.dst_port,
"requests": 0, "methods": set(), "uris": [], "clients": set(),
"user_agents": set(), "auth": False})
e["requests"] += 1
if r.method:
e["methods"].add(r.method)
if r.uri and len(e["uris"]) < 15 and r.uri not in e["uris"]:
e["uris"].append(r.uri)
if r.src_ip:
e["clients"].add(r.src_ip)
if r.user_agent:
e["user_agents"].add(r.user_agent[:100])
if r.authorization:
e["auth"] = True
out = []
for e in hosts.values():
e["methods"] = sorted(e["methods"])
e["clients"] = sorted(e["clients"])[:6]
e["user_agents"] = sorted(e["user_agents"])[:5]
e["device"] = bool(ctx.device_ip and (ctx.device_ip in e["clients"] or e["server_ip"] == ctx.device_ip))
out.append(e)
out.sort(key=lambda e: (-int(e["device"]), -e["requests"]))
status: dict[str, int] = defaultdict(int)
for resp in ctx.http_responses:
if resp.status_code is not None:
status[str(resp.status_code)] += 1
return {"requests": len(ctx.http_requests), "responses": len(ctx.http_responses),
"status_codes": dict(status), "hosts": out[:200],
"file_transfers": [
{"filename": f.filename, "extension": f.extension, "content_type": f.content_type,
"size": f.size, "protocol": f.protocol, "direction": f.direction,
"src_ip": f.src_ip, "dst_ip": f.dst_ip, "uri": f.uri, "packet": f.packet_number}
for f in ctx.file_transfers[:100]
]}
def _cleartext_section(ctx: CaptureContext) -> dict:
"""Unencrypted application flows and clear-text services grouped by protocol."""
flows: dict[str, dict] = {}
for c in ctx.conversations:
if c.payload_bytes <= 0 or c.is_encrypted:
continue
protos = sorted(c.app_protocols & _CLEARTEXT_APP) or ([_PORT_NAMES[c.dst_port]] if c.dst_port in _PORT_NAMES and c.dst_port not in (53, 123, 67, 68, 5353, 1900, 137, 138) else [])
if not protos:
continue
for p in protos:
e = flows.setdefault(p, {"protocol": p, "flows": 0, "bytes": 0, "pairs": set(), "device": False})
e["flows"] += 1
e["bytes"] += c.payload_bytes
e["pairs"].add(f"{c.src_ip} -> {c.dst_ip}:{c.dst_port}")
if ctx.is_device(c.src_ip) or ctx.is_device(c.dst_ip):
e["device"] = True
out = []
for e in flows.values():
e["pairs"] = sorted(e["pairs"])[:20]
out.append(e)
out.sort(key=lambda e: (-int(e["device"]), -e["bytes"]))
services = [
{"protocol": s.protocol, "kind": s.kind, "value": s.value, "detail": s.detail,
"src_ip": s.src_ip, "dst_ip": s.dst_ip, "port": s.dst_port, "packet": s.packet_number}
for s in ctx.service_events[:200]
]
return {"flows": out, "service_events": services,
"mqtt_events": len(ctx.mqtt_events),
"dhcp": [
{"type": d.message_type, "client_mac": d.client_mac, "client_ip": d.client_ip,
"hostname": d.hostname, "vendor_class": d.vendor_class, "server": d.server_id}
for d in ctx.dhcp_events[:60]
]}
def _coverage_section(ctx: CaptureContext) -> dict:
notes = list(ctx.coverage_notes)
if ctx.truncated:
notes.insert(0, ctx.truncation_reason or "Extraction was truncated.")
encrypted_flows = sum(1 for c in ctx.conversations if c.is_encrypted)
payload_flows = sum(1 for c in ctx.conversations if c.payload_bytes > 0)
undecoded = [c for c in ctx.conversations if c.payload_bytes > 0 and not c.app_protocols]
undecoded_ports: dict[int, int] = defaultdict(int)
for c in undecoded:
if c.dst_port:
undecoded_ports[c.dst_port] += 1
if not ctx.device_ip:
notes.append("No device address was configured; findings are not attributed to a device under test.")
tls13 = sum(1 for t in ctx.tls_sessions if t.version == "TLS 1.3")
if tls13:
notes.append(f"{tls13} TLS 1.3 sessions: server certificates are encrypted and could not be inspected.")
return {
"complete": not ctx.truncated,
"notes": notes,
"flows_total": len(ctx.conversations),
"flows_with_payload": payload_flows,
"flows_encrypted": encrypted_flows,
"flows_undecoded": len(undecoded),
"undecoded_ports": dict(sorted(undecoded_ports.items(), key=lambda kv: -kv[1])[:15]),
"text_payloads_scanned": len(ctx.text_payloads),
}