250 lines
8.1 KiB
Python
250 lines
8.1 KiB
Python
"""Normalized capture data consumed by every detector.
|
|
|
|
The module is independent of Django and pyshark so detector logic can be tested
|
|
with synthetic data. ``extract_pyshark.py`` performs the actual PCAP extraction.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Normalized records
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
@dataclass
|
|
class Host:
|
|
ip: str
|
|
macs: set[str] = field(default_factory=set)
|
|
hostnames: set[str] = field(default_factory=set)
|
|
bytes_sent: int = 0
|
|
bytes_received: int = 0
|
|
ports_offered: set[int] = field(default_factory=set) # Ports on which the host responds
|
|
ports_contacted: set[int] = field(default_factory=set) # Ports contacted by the host
|
|
|
|
@property
|
|
def is_private(self) -> bool:
|
|
return _is_private(self.ip)
|
|
|
|
|
|
@dataclass
|
|
class Conversation:
|
|
"""A five-tuple flow aggregated between two endpoints."""
|
|
|
|
src_ip: str
|
|
dst_ip: str
|
|
src_port: Optional[int]
|
|
dst_port: Optional[int]
|
|
protocol: str # For example, "tcp" or "udp"
|
|
packets: int = 0
|
|
bytes: int = 0
|
|
bytes_src_to_dst: int = 0
|
|
bytes_dst_to_src: int = 0
|
|
first_seen: Optional[float] = None
|
|
last_seen: Optional[float] = None
|
|
stream_id: Optional[int] = None
|
|
timestamps: list[float] = field(default_factory=list) # Used to detect beaconing
|
|
packet_numbers: list[int] = field(default_factory=list) # Bounded evidence packet list
|
|
|
|
|
|
@dataclass
|
|
class DnsQuery:
|
|
query_name: str
|
|
qtype: Optional[str] = None
|
|
response_ips: list[str] = field(default_factory=list)
|
|
rcode: Optional[str] = None # For example, "NOERROR" or "NXDOMAIN"
|
|
timestamp: Optional[float] = None
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
resolver: Optional[str] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class HttpRequest:
|
|
method: Optional[str] = None
|
|
host: Optional[str] = None
|
|
uri: Optional[str] = None
|
|
user_agent: Optional[str] = None
|
|
authorization: Optional[str] = None # Authorization header value
|
|
cookies: Optional[str] = None
|
|
headers: dict[str, str] = field(default_factory=dict)
|
|
body: Optional[str] = None
|
|
request_bytes: int = 0
|
|
is_tls: bool = False # True when only TLS metadata is visible
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
dst_port: Optional[int] = None
|
|
timestamp: Optional[float] = None
|
|
stream_id: Optional[int] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class HttpResponse:
|
|
status_code: Optional[int] = None
|
|
content_type: Optional[str] = None
|
|
server: Optional[str] = None
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
timestamp: Optional[float] = None
|
|
stream_id: Optional[int] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class TlsSession:
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
dst_port: Optional[int] = 443
|
|
version: Optional[str] = None # For example, "TLS 1.2" or "TLS 1.0"
|
|
cipher: Optional[str] = None # Cipher-suite name
|
|
sni: Optional[str] = None
|
|
cert_subject_cn: Optional[str] = None
|
|
cert_issuer_cn: Optional[str] = None
|
|
cert_not_after: Optional[str] = None
|
|
cert_self_signed: Optional[bool] = None
|
|
cert_expired: Optional[bool] = None
|
|
timestamp: Optional[float] = None
|
|
stream_id: Optional[int] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class AuthMaterial:
|
|
"""Hashes, challenge-response material, and authentication tickets."""
|
|
|
|
kind: str # "ntlmssp", "netntlmv2", "kerberos_asrep", ...
|
|
value: Optional[str] = None
|
|
account: Optional[str] = None
|
|
domain: Optional[str] = None
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
protocol: Optional[str] = None
|
|
timestamp: Optional[float] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
# Crackable material, when reconstruction is possible.
|
|
hashcat: Optional[str] = None # Complete hashcat input line
|
|
hashcat_mode: Optional[int] = None # For example, 5600 (NetNTLMv2)
|
|
hash_format: Optional[str] = None # Human-readable format name
|
|
server_challenge: Optional[str] = None # NTLMSSP server challenge (hex)
|
|
nt_response: Optional[str] = None # NTLMv2 response (hex)
|
|
etype: Optional[str] = None # Kerberos encryption type
|
|
|
|
|
|
@dataclass
|
|
class ArpEvent:
|
|
sender_ip: Optional[str] = None
|
|
sender_mac: Optional[str] = None
|
|
target_ip: Optional[str] = None
|
|
opcode: Optional[int] = None # 1=request, 2=reply
|
|
is_gratuitous: bool = False
|
|
timestamp: Optional[float] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class FileTransfer:
|
|
filename: Optional[str] = None
|
|
extension: Optional[str] = None
|
|
content_type: Optional[str] = None
|
|
size: Optional[int] = None
|
|
protocol: Optional[str] = None # "http", "ftp-data", "smb", "tftp"
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
direction: Optional[str] = None # "download" / "upload"
|
|
timestamp: Optional[float] = None
|
|
stream_id: Optional[int] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class PlaintextAuth:
|
|
"""Clear-text authentication observed in an application protocol."""
|
|
|
|
protocol: str
|
|
username: Optional[str] = None
|
|
password: Optional[str] = None
|
|
src_ip: Optional[str] = None
|
|
dst_ip: Optional[str] = None
|
|
dst_port: Optional[int] = None
|
|
timestamp: Optional[float] = None
|
|
stream_id: Optional[int] = None
|
|
packet_number: Optional[int] = None
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Central capture context
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
@dataclass
|
|
class CaptureContext:
|
|
"""All detector input extracted during a single capture pass."""
|
|
|
|
path: Optional[str] = None
|
|
packet_count: int = 0
|
|
start_time: Optional[float] = None
|
|
end_time: Optional[float] = None
|
|
|
|
hosts: dict[str, Host] = field(default_factory=dict)
|
|
conversations: list[Conversation] = field(default_factory=list)
|
|
dns_queries: list[DnsQuery] = field(default_factory=list)
|
|
http_requests: list[HttpRequest] = field(default_factory=list)
|
|
http_responses: list[HttpResponse] = field(default_factory=list)
|
|
tls_sessions: list[TlsSession] = field(default_factory=list)
|
|
auth_material: list[AuthMaterial] = field(default_factory=list)
|
|
arp_events: list[ArpEvent] = field(default_factory=list)
|
|
file_transfers: list[FileTransfer] = field(default_factory=list)
|
|
plaintext_auth: list[PlaintextAuth] = field(default_factory=list)
|
|
|
|
# protocol_stats: protocol-name -> packet count
|
|
protocol_stats: dict[str, int] = field(default_factory=dict)
|
|
# Generic text payloads with packet metadata for secret detection.
|
|
text_payloads: list[dict] = field(default_factory=list)
|
|
|
|
@property
|
|
def duration(self) -> float:
|
|
if self.start_time is None or self.end_time is None:
|
|
return 0.0
|
|
return max(0.0, self.end_time - self.start_time)
|
|
|
|
def host(self, ip: str) -> Host:
|
|
h = self.hosts.get(ip)
|
|
if h is None:
|
|
h = Host(ip=ip)
|
|
self.hosts[ip] = h
|
|
return h
|
|
|
|
def is_external(self, ip: Optional[str]) -> bool:
|
|
"""Return whether an address is globally routable."""
|
|
if not ip:
|
|
return False
|
|
return not _is_private(ip)
|
|
|
|
def protocol_count(self, *names: str) -> int:
|
|
"""Return a case-insensitive packet total for the requested protocols."""
|
|
wanted = {n.lower() for n in names}
|
|
return sum(
|
|
count for proto, count in self.protocol_stats.items()
|
|
if str(proto).lower() in wanted
|
|
)
|
|
|
|
|
|
def _is_private(ip: str) -> bool:
|
|
try:
|
|
addr = ipaddress.ip_address(ip)
|
|
except ValueError:
|
|
return False
|
|
return (
|
|
addr.is_private
|
|
or addr.is_loopback
|
|
or addr.is_link_local
|
|
or addr.is_multicast
|
|
or addr.is_reserved
|
|
or addr.is_unspecified
|
|
)
|