"""Build a normalized capture context with tshark through pyshark. pyshark is loaded lazily, and individual packet handlers are isolated so one unexpected packet cannot abort the complete analysis. """ from __future__ import annotations import base64 import os from typing import Optional from .context import ( CaptureContext, Conversation, DnsQuery, HttpRequest, HttpResponse, TlsSession, ArpEvent, PlaintextAuth, AuthMaterial, FileTransfer, ) from . import crackformats # Known clear-text protocols used by extraction heuristics. CLEARTEXT_PORTS = { 21: "ftp", 23: "telnet", 25: "smtp", 80: "http", 110: "pop3", 143: "imap", 389: "ldap", 161: "snmp", 69: "tftp", } def build_context_from_pcap(path: str, packet_limit: Optional[int] = None) -> CaptureContext: """Read a capture and return its normalized context.""" if not os.path.exists(path): raise FileNotFoundError(f"Capture not found: {path}") try: import pyshark # lazy import except ImportError as exc: # pragma: no cover raise RuntimeError( "pyshark or tshark is unavailable. Install tshark or use the backend container." ) from exc ctx = CaptureContext(path=path) cap = pyshark.FileCapture(path, keep_packets=False) conv_index: dict[tuple, Conversation] = {} # Transient state correlates NTLM and SMTP authentication across packets. state: dict = {"ntlm_chal": {}, "smtp_auth": {}} n = 0 try: for pkt in cap: n += 1 if packet_limit and n > packet_limit: break try: _handle_packet(ctx, pkt, conv_index, state) except Exception: # Skip isolated malformed or unexpected packets. continue except Exception as exc: if exc.__class__.__name__ == "TSharkNotFoundException": raise RuntimeError( "tshark was not found. Install Wireshark/tshark or use the backend container." ) from exc raise finally: try: cap.close() except Exception: pass ctx.packet_count = n ctx.conversations = list(conv_index.values()) return ctx # --------------------------------------------------------------------------- # # Packet handlers # --------------------------------------------------------------------------- # def _ts(pkt) -> Optional[float]: try: return float(pkt.sniff_timestamp) except Exception: return None def _num(pkt) -> Optional[int]: try: return int(pkt.number) except Exception: return None def _handle_packet(ctx: CaptureContext, pkt, conv_index: dict, state: Optional[dict] = None) -> None: if state is None: state = {"ntlm_chal": {}, "smtp_auth": {}} ts = _ts(pkt) if ts is not None: ctx.start_time = ts if ctx.start_time is None else min(ctx.start_time, ts) ctx.end_time = ts if ctx.end_time is None else max(ctx.end_time, ts) # Count packets by the highest decoded protocol layer. try: top = pkt.highest_layer ctx.protocol_stats[top] = ctx.protocol_stats.get(top, 0) + 1 except Exception: pass layers = {l.layer_name for l in pkt.layers} src_ip = dst_ip = None length = 0 try: length = int(pkt.length) except Exception: pass if "ip" in layers: try: src_ip = pkt.ip.src dst_ip = pkt.ip.dst except Exception: pass elif "ipv6" in layers: try: src_ip = pkt.ipv6.src dst_ip = pkt.ipv6.dst except Exception: pass # MAC-Adressen / Hosts src_mac = dst_mac = None if "eth" in layers: try: src_mac, dst_mac = pkt.eth.src, pkt.eth.dst except Exception: pass src_port = dst_port = None transport = None if "tcp" in layers: transport = "tcp" try: src_port = int(pkt.tcp.srcport) dst_port = int(pkt.tcp.dstport) except Exception: pass elif "udp" in layers: transport = "udp" try: src_port = int(pkt.udp.srcport) dst_port = int(pkt.udp.dstport) except Exception: pass if src_ip: h = ctx.host(src_ip) h.bytes_sent += length if src_mac: h.macs.add(src_mac) if dst_port: h.ports_contacted.add(dst_port) if dst_ip: h = ctx.host(dst_ip) h.bytes_received += length if dst_port: h.ports_offered.add(dst_port) # Conversation aggregieren if src_ip and dst_ip and transport: key = tuple(sorted([(src_ip, src_port), (dst_ip, dst_port)]) + [transport]) conv = conv_index.get(key) if conv is None: conv = Conversation( src_ip=src_ip, dst_ip=dst_ip, src_port=src_port, dst_port=dst_port, protocol=transport, first_seen=ts, last_seen=ts, ) try: conv.stream_id = int(pkt.tcp.stream) if transport == "tcp" else None except Exception: conv.stream_id = None conv_index[key] = conv conv.packets += 1 conv.bytes += length pnum = _num(pkt) if pnum is not None and len(conv.packet_numbers) < 20: conv.packet_numbers.append(pnum) if ts is not None: conv.timestamps.append(ts) conv.last_seen = ts if conv.last_seen is None else max(conv.last_seen, ts) conv.first_seen = ts if conv.first_seen is None else min(conv.first_seen, ts) if src_ip == conv.src_ip: conv.bytes_src_to_dst += length else: conv.bytes_dst_to_src += length # TCP stream IDs support cross-packet authentication correlation. stream_id = None if transport == "tcp": try: stream_id = int(pkt.tcp.stream) except Exception: stream_id = None # Protocol-specific extraction. if "arp" in layers: _handle_arp(ctx, pkt, ts) if "dns" in layers: _handle_dns(ctx, pkt, src_ip, dst_ip, ts) if "http" in layers: _handle_http(ctx, pkt, src_ip, dst_ip, dst_port, ts) if "tls" in layers or "ssl" in layers: _handle_tls(ctx, pkt, src_ip, dst_ip, dst_port, ts) if "ftp" in layers: _handle_ftp(ctx, pkt, src_ip, dst_ip, dst_port, ts) if "telnet" in layers: _handle_telnet(ctx, pkt, src_ip, dst_ip, dst_port, ts) if "pop" in layers: _handle_pop3(ctx, pkt, src_ip, dst_ip, dst_port, ts) if "imap" in layers: _handle_imap(ctx, pkt, src_ip, dst_ip, dst_port, ts) if "smtp" in layers: _handle_smtp(ctx, pkt, src_ip, dst_ip, dst_port, ts, stream_id, state) if "ntlmssp" in layers: _handle_ntlm(ctx, pkt, src_ip, dst_ip, ts, stream_id, state) if "kerberos" in layers: _handle_kerberos(ctx, pkt, src_ip, dst_ip, ts) # Generic payload extraction finds artifacts that protocol fields may omit. _handle_raw_payload(ctx, pkt, layers, src_ip, dst_ip, ts) def _handle_arp(ctx, pkt, ts): try: a = pkt.arp ctx.arp_events.append(ArpEvent( sender_ip=getattr(a, "src_proto_ipv4", None), sender_mac=getattr(a, "src_hw_mac", None), target_ip=getattr(a, "dst_proto_ipv4", None), opcode=int(getattr(a, "opcode", 0) or 0), timestamp=ts, packet_number=_num(pkt), )) except Exception: pass def _handle_dns(ctx, pkt, src_ip, dst_ip, ts): try: d = pkt.dns # Ignore DNS records without a query name. qname = getattr(d, "qry_name", None) if not qname: return rcode = getattr(d, "flags_rcode", None) rcode_map = {"0": "NOERROR", "3": "NXDOMAIN"} response_ips = [] for attr in ("a", "aaaa"): response_ips.extend(_all_field_values(d, attr)) ctx.dns_queries.append(DnsQuery( query_name=qname, qtype=_dns_qtype_name(getattr(d, "qry_type", None)), response_ips=response_ips, rcode=rcode_map.get(str(rcode), rcode), timestamp=ts, src_ip=src_ip, dst_ip=dst_ip, packet_number=_num(pkt), )) except Exception: pass def _handle_http(ctx, pkt, src_ip, dst_ip, dst_port, ts): try: h = pkt.http if hasattr(h, "request_method") or hasattr(h, "request_uri"): req = HttpRequest( method=getattr(h, "request_method", None), host=getattr(h, "host", None), uri=getattr(h, "request_uri", None), user_agent=getattr(h, "user_agent", None), authorization=getattr(h, "authorization", None), cookies=getattr(h, "cookie", None), body=getattr(h, "file_data", None), src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port, timestamp=ts, packet_number=_num(pkt), ) ctx.http_requests.append(req) # Preserve visible HTTP text for artifact detection. for piece in (req.uri, req.cookies, req.authorization, req.body): if piece: ctx.text_payloads.append({ "text": str(piece), "protocol": "http", "src_ip": src_ip, "dst_ip": dst_ip, "packet_number": _num(pkt), "timestamp": ts, }) if hasattr(h, "response_code"): ct = getattr(h, "content_type", None) ctx.http_responses.append(HttpResponse( status_code=_safe_int(getattr(h, "response_code", None)), content_type=ct, server=getattr(h, "server", None), src_ip=src_ip, dst_ip=dst_ip, timestamp=ts, packet_number=_num(pkt), )) # Record only responses that look like file downloads. disposition = getattr(h, "content_disposition", None) filename = _filename_from_disposition(disposition) if filename or _is_download_content_type(ct): ext = os.path.splitext(filename or "")[1].lower() or None ctx.file_transfers.append(FileTransfer( filename=filename, extension=ext, content_type=ct, protocol="http", src_ip=src_ip, dst_ip=dst_ip, direction="download", timestamp=ts, packet_number=_num(pkt), )) except Exception: pass def _handle_tls(ctx, pkt, src_ip, dst_ip, dst_port, ts): try: layer = getattr(pkt, "tls", None) or getattr(pkt, "ssl", None) if layer is None: return # TLS 1.3 reports its real version in the supported-versions extension. version = ( getattr(layer, "handshake_extensions_supported_version", None) or getattr(layer, "handshake_version", None) or getattr(layer, "record_version", None) ) subject_cn = _first( getattr(layer, "x509sat_printablestring", None), getattr(layer, "x509ce_dnsname", None), ) issuer_cn = getattr(layer, "x509sat_printablestring_1", None) not_after = ( getattr(layer, "x509af_utctime", None) or getattr(layer, "x509af_notafter", None) or getattr(layer, "x509af_utctime_1", None) ) ctx.tls_sessions.append(TlsSession( src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port or 443, version=_tls_version_name(version), cipher=getattr(layer, "handshake_ciphersuite", None), sni=getattr(layer, "handshake_extensions_server_name", None), cert_subject_cn=subject_cn, cert_issuer_cn=issuer_cn, cert_not_after=not_after, cert_self_signed=_cert_is_self_signed(subject_cn, issuer_cn), cert_expired=_cert_is_expired(not_after, ctx.end_time or ts), timestamp=ts, packet_number=_num(pkt), )) except Exception: pass def _handle_ftp(ctx, pkt, src_ip, dst_ip, dst_port, ts): try: f = pkt.ftp cmd = getattr(f, "request_command", None) arg = getattr(f, "request_arg", None) if cmd and cmd.upper() in ("USER", "PASS"): pa = _last_plaintext(ctx, "ftp", src_ip, dst_ip, dst_port) if cmd.upper() == "USER": pa.username = arg else: pa.password = arg pa.timestamp = ts pa.packet_number = _num(pkt) except Exception: pass def _handle_telnet(ctx, pkt, src_ip, dst_ip, dst_port, ts): try: data = getattr(pkt.telnet, "data", None) if data: ctx.text_payloads.append({ "text": str(data), "protocol": "telnet", "src_ip": src_ip, "dst_ip": dst_ip, "packet_number": _num(pkt), "timestamp": ts, }) except Exception: pass def _handle_pop3(ctx, pkt, src_ip, dst_ip, dst_port, ts): """POP3 USER/PASS im Klartext.""" try: p = pkt.pop cmd = (getattr(p, "request_command", None) or "").upper() param = getattr(p, "request_parameter", None) if cmd in ("USER", "PASS") and param: pa = _last_plaintext(ctx, "pop3", src_ip, dst_ip, dst_port) if cmd == "USER": pa.username = param else: pa.password = param pa.timestamp = ts pa.packet_number = _num(pkt) except Exception: pass def _handle_imap(ctx, pkt, src_ip, dst_ip, dst_port, ts): """Extract clear-text IMAP LOGIN credentials.""" try: i = pkt.imap line = getattr(i, "request", None) or getattr(i, "request_command", None) if not line: return parts = str(line).split() # Accept both tagged and untagged IMAP LOGIN forms. idx = next((k for k, tok in enumerate(parts) if tok.upper() == "LOGIN"), None) if idx is not None and len(parts) >= idx + 3: user = parts[idx + 1].strip('"') pw = parts[idx + 2].strip('"') pa = _last_plaintext(ctx, "imap", src_ip, dst_ip, dst_port) pa.username, pa.password = user, pw pa.timestamp = ts pa.packet_number = _num(pkt) except Exception: pass def _handle_smtp(ctx, pkt, src_ip, dst_ip, dst_port, ts, stream_id, state): """Extract Base64-encoded SMTP AUTH LOGIN or PLAIN credentials.""" try: s = pkt.smtp param = getattr(s, "req_parameter", None) or getattr(s, "request_parameter", None) cmd = (getattr(s, "req_command", None) or getattr(s, "request_command", None) or "").upper() if not param and not cmd: return pending = state["smtp_auth"].setdefault(stream_id, {"mech": None, "tokens": []}) text = f"{cmd} {param}".strip() upper = text.upper() if "AUTH LOGIN" in upper: pending["mech"] = "login" return if "AUTH PLAIN" in upper: pending["mech"] = "plain" # AUTH PLAIN may carry its token on the command line. tail = text.split() if len(tail) >= 3: creds = crackformats.decode_smtp_auth_plain(tail[2]) _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt) state["smtp_auth"].pop(stream_id, None) return # AUTH LOGIN sends username and password in subsequent lines. if pending["mech"] == "login" and param: pending["tokens"].append(param) if len(pending["tokens"]) >= 2: user, pw = crackformats.decode_smtp_auth_login( pending["tokens"][0], pending["tokens"][1]) _emit_smtp(ctx, (user, pw), src_ip, dst_ip, dst_port, ts, pkt) state["smtp_auth"].pop(stream_id, None) elif pending["mech"] == "plain" and param: creds = crackformats.decode_smtp_auth_plain(param) _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt) state["smtp_auth"].pop(stream_id, None) except Exception: pass def _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt): if not creds: return user, pw = creds if not (user or pw): return ctx.plaintext_auth.append(PlaintextAuth( protocol="smtp", username=user, password=pw, src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port, timestamp=ts, packet_number=_num(pkt), )) def _handle_ntlm(ctx, pkt, src_ip, dst_ip, ts, stream_id, state): """Correlate NTLM challenge and response packets into hashcat material.""" try: n = pkt.ntlmssp msg_type = str(getattr(n, "messagetype", "") or "").lower() # Retain the type-2 server challenge for the matching type-3 response. server_chal = getattr(n, "ntlmserverchallenge", None) if server_chal and ("0x00000002" in msg_type or msg_type.endswith("2")): state["ntlm_chal"][stream_id] = str(server_chal) # A type-3 message provides the account and NT response. account = getattr(n, "auth_username", None) domain = getattr(n, "auth_domain", None) nt_resp = (getattr(n, "ntlmv2_response", None) or getattr(n, "auth_ntresponse", None)) if account or nt_resp: chal = state["ntlm_chal"].get(stream_id) am = AuthMaterial( kind="ntlmssp", account=account, domain=domain, src_ip=src_ip, dst_ip=dst_ip, protocol="ntlm", timestamp=ts, packet_number=_num(pkt), server_challenge=chal, nt_response=str(nt_resp) if nt_resp else None, ) built = crackformats.build_netntlmv2(account, domain, chal, nt_resp) if built: am.kind = "netntlmv2" am.hashcat = built["hashcat"] am.hashcat_mode = built["hashcat_mode"] am.hash_format = built["format"] ctx.auth_material.append(am) except Exception: pass # Bound retained payload text for large captures. _MAX_TEXT_PAYLOADS = 20000 def _handle_raw_payload(ctx, pkt, layers, src_ip, dst_ip, ts): """Retain bounded TCP or UDP text for generic artifact detection.""" if len(ctx.text_payloads) >= _MAX_TEXT_PAYLOADS: return raw = None try: if "tcp" in layers and hasattr(pkt.tcp, "payload"): raw = pkt.tcp.payload elif "udp" in layers and hasattr(pkt.udp, "payload"): raw = pkt.udp.payload elif "data" in layers: raw = getattr(pkt.data, "data", None) except Exception: raw = None if not raw: return text = _bytes_from_hexish(str(raw)) if not text or not text.strip(): return try: proto = (pkt.highest_layer or "raw").lower() except Exception: proto = "raw" ctx.text_payloads.append({ "text": text[:16384], "protocol": proto, "src_ip": src_ip, "dst_ip": dst_ip, "packet_number": _num(pkt), "timestamp": ts, }) def _bytes_from_hexish(raw: str) -> Optional[str]: """Convert a hex-formatted pyshark payload to text when possible.""" s = str(raw).strip() compact = s.replace(":", "").replace(".", "").replace(" ", "") if compact and len(compact) % 2 == 0 and all(c in "0123456789abcdefABCDEF" for c in compact): try: return bytes.fromhex(compact).decode("utf-8", "ignore") except Exception: return None # Some dissectors already expose decoded text. return s def _handle_kerberos(ctx, pkt, src_ip, dst_ip, ts): """Extract Kerberos AS-REP and TGS-REP material for offline analysis.""" try: k = pkt.kerberos msg_type = str(getattr(k, "msg_type", "") or "") account = getattr(k, "cnamestring", None) or getattr(k, "cnamestring_1", None) realm = getattr(k, "realm", None) or getattr(k, "crealm", None) etype = getattr(k, "etype", None) cipher = getattr(k, "cipher", None) or getattr(k, "enc_part_cipher", None) am = AuthMaterial( kind=f"kerberos_{msg_type}" if msg_type else "kerberos", account=account, domain=realm, src_ip=src_ip, dst_ip=dst_ip, protocol="kerberos", timestamp=ts, packet_number=_num(pkt), etype=str(etype) if etype is not None else None, ) # Kerberos message types 11 and 13 are AS-REP and TGS-REP. built = None if msg_type == "11": built = crackformats.build_krb5_asrep(account, realm, etype, cipher) elif msg_type == "13": spn = getattr(k, "snamestring", None) built = crackformats.build_krb5_tgsrep(account, realm, spn, etype, cipher) if built: am.hashcat = built["hashcat"] am.hashcat_mode = built["hashcat_mode"] am.hash_format = built["format"] ctx.auth_material.append(am) except Exception: pass # --------------------------------------------------------------------------- # # kleine Helfer # --------------------------------------------------------------------------- # def _last_plaintext(ctx, protocol, src_ip, dst_ip, dst_port) -> PlaintextAuth: """Find or create the authentication record that combines USER and PASS.""" for pa in reversed(ctx.plaintext_auth): if pa.protocol == protocol and pa.src_ip == src_ip and pa.dst_ip == dst_ip: if pa.password is None or pa.username is None: return pa pa = PlaintextAuth(protocol=protocol, src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port) ctx.plaintext_auth.append(pa) return pa def _safe_int(v): try: return int(v) except Exception: return None def _tls_version_name(raw) -> Optional[str]: mapping = { "0x0300": "SSL 3.0", "0x0301": "TLS 1.0", "0x0302": "TLS 1.1", "0x0303": "TLS 1.2", "0x0304": "TLS 1.3", "768": "SSL 3.0", "769": "TLS 1.0", "770": "TLS 1.1", "771": "TLS 1.2", "772": "TLS 1.3", } if raw is None: return None return mapping.get(str(raw).lower(), str(raw)) def _first(*vals): """Return the first non-empty dissector field value.""" for v in vals: if v: return v return None def _all_field_values(layer, name) -> list[str]: """Return every occurrence of a repeated pyshark field.""" out: list[str] = [] try: field = getattr(layer, name, None) if field is None: return out allf = getattr(field, "all_fields", None) if allf: for f in allf: v = getattr(f, "show", None) or getattr(f, "showname_value", None) if v: out.append(str(v)) else: out.append(str(field)) except Exception: pass return out # tshark often exposes DNS query types as numeric values. _DNS_QTYPES = { "1": "A", "2": "NS", "5": "CNAME", "6": "SOA", "12": "PTR", "15": "MX", "16": "TXT", "28": "AAAA", "33": "SRV", "255": "ANY", "10": "NULL", "43": "DS", "48": "DNSKEY", "257": "CAA", "65": "HTTPS", "64": "SVCB", } def _dns_qtype_name(raw) -> Optional[str]: if raw is None: return None s = str(raw).strip() return _DNS_QTYPES.get(s, s.upper()) # Content types that indicate a file rather than ordinary web content. _DOWNLOAD_CT_MARKERS = ( "application/octet-stream", "application/x-msdownload", "application/x-dosexec", "application/x-executable", "application/vnd.microsoft.portable-executable", "application/zip", "application/x-rar", "application/x-7z", "application/x-tar", "application/gzip", "application/x-msi", "application/x-sh", "application/java-archive", "application/x-elf", ) def _is_download_content_type(ct) -> bool: if not ct: return False low = str(ct).lower() return any(m in low for m in _DOWNLOAD_CT_MARKERS) def _filename_from_disposition(disposition) -> Optional[str]: """Extract a filename from a Content-Disposition header.""" if not disposition: return None import re as _re m = _re.search(r'filename\*?=(?:"([^"]+)"|([^;]+))', str(disposition), _re.IGNORECASE) if not m: return None name = (m.group(1) or m.group(2) or "").strip().strip('"') # Remove an optional RFC 5987 character-set prefix. if "''" in name: name = name.split("''", 1)[1] return name or None def _cert_is_self_signed(subject_cn, issuer_cn) -> Optional[bool]: """Return whether subject and issuer match, or None if data is incomplete.""" if not subject_cn or not issuer_cn: return None return str(subject_cn).strip().lower() == str(issuer_cn).strip().lower() def _cert_is_expired(not_after, now_epoch) -> Optional[bool]: """Compare a parsed certificate expiry with the capture timestamp.""" if not not_after or now_epoch is None: return None exp = _parse_cert_time(not_after) if exp is None: return None try: return float(now_epoch) > exp except (TypeError, ValueError): return None def _parse_cert_time(raw) -> Optional[float]: """Convert common X.509 time formats to a UTC epoch timestamp.""" import datetime as _dt s = str(raw).strip() # tshark may already provide an epoch value. try: return float(s) except ValueError: pass fmts = ( "%Y-%m-%d %H:%M:%S", # 2020-01-01 00:00:00 "%b %d %H:%M:%S %Y GMT", # Jan 1 00:00:00 2020 GMT "%y%m%d%H%M%SZ", # UTCTime 200101000000Z "%Y%m%d%H%M%SZ", # GeneralizedTime "%Y-%m-%dT%H:%M:%SZ", # ISO 8601 ) for fmt in fmts: try: dt = _dt.datetime.strptime(s, fmt).replace(tzinfo=_dt.timezone.utc) return dt.timestamp() except ValueError: continue return None