analyze performance upgrade

django_project/analyzer/artifacts.py

@@ -1,6 +1,6 @@
 """Extract endpoint and crackable hash artifacts from a capture context.
 
-Extraction is independent of Django, pyshark, and I/O. Active endpoint probing
+Extraction is independent of Django, tshark, and I/O. Active endpoint probing
 and hash cracking remain isolated in ``analyzer.enrich``.
 """
 
@@ -143,6 +143,38 @@ _URL_RE = re.compile(
 )
 
 
+# Endpoints are only trustworthy from cleartext application data. Scanning
+# encrypted/binary payloads (TLS, QUIC, SSH, raw TCP) with a loose domain regex
+# invents host-shaped noise from random bytes, so payload extraction is limited
+# to protocols whose bytes are actually text.
+_CLEARTEXT_PROTOS = {
+    "http", "http2", "telnet", "ftp", "ftp-data", "ftp_data", "smtp", "pop",
+    "pop3", "imap", "snmp", "tftp", "ldap", "sip", "irc", "syslog", "rtsp",
+    "xmpp", "mqtt", "urlencoded-form", "json", "xml", "data-text-lines",
+}
+
+_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+
+
+def _looks_like_host(value: str) -> bool:
+    """Reject byte noise that the URL regex mistakes for a bare domain."""
+    v = value.split("://")[-1].split("/")[0]
+    if _IPV4_RE.match(v):
+        return all(0 <= int(o) <= 255 for o in v.split("."))
+    if len(v) < 4 or ".." in v:
+        return False
+    labels = v.split(".")
+    if len(labels) < 2:
+        return False
+    tld = labels[-1]
+    if not tld.isalpha() or not (2 <= len(tld) <= 24):
+        return False
+    # At least one substantive label — kills noise like "4v.b.di" / "q.p.fh".
+    if not any(len(lbl) >= 3 for lbl in labels[:-1]):
+        return False
+    return True
+
+
 def _normalize_host(value: Optional[str]) -> Optional[str]:
     if not value:
         return None
@@ -187,8 +219,12 @@ def extract_urls(ctx: CaptureContext) -> list[UrlArtifact]:
                 art.observed_ports.add(r.dst_port)
 
     for p in ctx.text_payloads:
+        if (p.get("protocol") or "").lower() not in _CLEARTEXT_PROTOS:
+            continue
         text = p.get("text") or ""
         for m in _URL_RE.findall(text):
+            if not _looks_like_host(m):
+                continue
             add(m, "payload", src_ip=p.get("src_ip"), dst_ip=p.get("dst_ip"),
                 timestamp=p.get("timestamp"), packet_number=p.get("packet_number"))
 
@@ -226,7 +262,11 @@ def extract_hashes(ctx: CaptureContext) -> list[HashArtifact]:
                 timestamp=am.timestamp, packet_number=am.packet_number)
 
     # Raw 32–64 character hex values use a mode only when the length is known.
+    # Only cleartext payloads are scanned; TLS/QUIC ciphertext is full of 32–64
+    # char hex runs that are not hashes.
     for p in ctx.text_payloads:
+        if (p.get("protocol") or "").lower() not in _CLEARTEXT_PROTOS:
+            continue
         text = p.get("text") or ""
         for m in _HASH_HEX_RE.findall(text):
             mode, fmt = identify_hash_mode(m)