build 4.6 Demo

2026-08-19 08:04:59 +02:00
commit ff8f1ca989
123 changed files with 17696 additions and 0 deletions
@@ -0,0 +1,168 @@
+"""Optional hash cracking and endpoint probing with progress persistence."""
+
+from __future__ import annotations
+
+import logging
+import threading
+
+from django.db import connection
+
+logger = logging.getLogger(__name__)
+
+
+def start_deep_analysis(file_id: int, url_ids=None, hash_ids=None,
+                        synchronous: bool = False) -> None:
+    """Start enrichment for all artifacts or the explicitly selected raw IDs."""
+    if synchronous:
+        _run(file_id, url_ids, hash_ids, close_conn=False)
+        return
+    thread = threading.Thread(target=_run, args=(file_id, url_ids, hash_ids), daemon=True,
+                              name=f"deep-analysis-{file_id}")
+    thread.start()
+
+
+def _analyzed_dataset_for(raw_obj):
+    """Return the single analyzed dataset associated with a capture."""
+    from IoTAnalyzer.models import AnalyzedDataset
+    file_obj = raw_obj.raw_dataset.file
+    ds, _ = AnalyzedDataset.objects.get_or_create(file=file_obj)
+    return ds
+
+
+def crack_raw_hash(raw_hash) -> None:
+    """Crack one raw hash and persist the result idempotently."""
+    from django.db import transaction
+    from IoTAnalyzer.models import Tool, HashType, HashAnalyzed
+    from analyzer.artifacts import HashArtifact
+    from analyzer.enrich import crack_hash
+
+    try:
+        res = crack_hash(HashArtifact(value=raw_hash.hash_value))
+        cracked = bool(res.cracked and res.plaintext)
+        logger.info("Hash id=%s mode=%s state=%s cracked=%s error=%s",
+                    raw_hash.id, res.hashcat_mode, res.state, cracked, res.error)
+
+        ds = _analyzed_dataset_for(raw_hash)
+        tool, _ = Tool.objects.get_or_create(tool_name="Hashcat", defaults={"version": "6.x.x"})
+        htype, _ = HashType.objects.get_or_create(hash_type=(res.hash_format or "unknown"))
+        crack_time = f"{res.duration_seconds}s" if res.duration_seconds is not None else None
+
+        with transaction.atomic():
+            ha, created = HashAnalyzed.objects.get_or_create(
+                analyzed_dataset=ds, hash_value=res.value,
+                defaults={"tool": tool, "cracked": cracked,
+                          "cracked_value": res.plaintext if cracked else None,
+                          "crack_time": crack_time},
+            )
+            if not created:
+                ha.tool = tool
+                ha.cracked = cracked
+                ha.cracked_value = res.plaintext if cracked else None
+                ha.crack_time = crack_time
+                ha.save()
+            ha.hash_type.add(htype)
+        logger.info("Stored hash result id=%s (created=%s, cracked=%s)", ha.id, created, cracked)
+    except Exception as exc:
+        logger.exception("Hash enrichment failed (id=%s): %s", raw_hash.id, exc)
+
+
+def probe_raw_url(raw_url) -> None:
+    """Probe one raw endpoint and persist DNS, port, and TLS results."""
+    from django.db import transaction
+    from IoTAnalyzer.models import Tool, UrlAnalyzed, Port, TlsVersion, CipherSuite
+    from analyzer.enrich import probe_url
+
+    try:
+        probe = probe_url(raw_url.url, active=True)
+        logger.info("URL id=%s ports=%s tls=%s error=%s",
+                    raw_url.id, probe.open_ports, probe.tls_versions, probe.error)
+
+        ds = _analyzed_dataset_for(raw_url)
+        tool, _ = Tool.objects.get_or_create(tool_name="DNS Analyzer", defaults={"version": "1.0"})
+
+        with transaction.atomic():
+            ua, created = UrlAnalyzed.objects.get_or_create(
+                analyzed_dataset=ds, url=raw_url.url, defaults={"tool": tool})
+            for p in probe.open_ports:
+                port, _ = Port.objects.get_or_create(port=p)
+                ua.ports.add(port)
+            for v in (probe.tls_versions or ["N/A"]):
+                tls, _ = TlsVersion.objects.get_or_create(tls_version=v)
+                ua.tls_versions.add(tls)
+            for c in (probe.cipher_suites or ["N/A"]):
+                cs, _ = CipherSuite.objects.get_or_create(cipher_suite=c)
+                ua.cipher_suites.add(cs)
+        logger.info("Stored endpoint result id=%s (created=%s)", ua.id, created)
+    except Exception as exc:
+        logger.exception("Endpoint enrichment failed (id=%s): %s", raw_url.id, exc)
+
+
+def _run(file_id: int, url_ids=None, hash_ids=None, close_conn: bool = True) -> None:
+    from IoTAnalyzer.models import UrlRaw, HashRaw, HashAnalyzed, DeepAnalysisJob
+
+    job = None
+    try:
+        url_qs = UrlRaw.objects.filter(raw_dataset__file_id=file_id)
+        hash_qs = HashRaw.objects.filter(raw_dataset__file_id=file_id)
+        # None selects all records; an empty list intentionally selects none.
+        if url_ids is not None:
+            url_qs = url_qs.filter(id__in=url_ids)
+        if hash_ids is not None:
+            hash_qs = hash_qs.filter(id__in=hash_ids)
+        urls = list(url_qs)
+        hashes = list(hash_qs)
+
+        DeepAnalysisJob.objects.filter(file_id=file_id).delete()
+        job = DeepAnalysisJob.objects.create(
+            file_id=file_id, state='running',
+            url_total=len(urls), hash_total=len(hashes),
+            message='Enrichment started',
+        )
+        logger.info("Enrichment started: file=%s urls=%s hashes=%s",
+                    file_id, len(urls), len(hashes))
+
+        # Crack hashes first because they usually complete quickly.
+        for raw_hash in hashes:
+            try:
+                crack_raw_hash(raw_hash)
+            except Exception as exc:
+                logger.exception("Hash cracking failed (id=%s): %s", raw_hash.id, exc)
+            job.hash_done += 1
+            job.message = f"Hash-Cracking {job.hash_done}/{job.hash_total}"
+            job.save(update_fields=['hash_done', 'message', 'updated'])
+
+        # Endpoint probing may perform DNS, port, and TLS checks.
+        for raw_url in urls:
+            try:
+                probe_raw_url(raw_url)
+            except Exception as exc:
+                logger.exception("Endpoint probing failed (id=%s): %s", raw_url.id, exc)
+            job.url_done += 1
+            job.message = f"Endpoint probing {job.url_done}/{job.url_total}"
+            job.save(update_fields=['url_done', 'message', 'updated'])
+
+        try:
+            job.cracked = HashAnalyzed.objects.filter(
+                analyzed_dataset__file_id=file_id, cracked=True
+            ).count()
+        except Exception:
+            pass
+
+        job.state = 'done'
+        job.message = 'Enrichment completed'
+        job.save()
+        logger.info("Enrichment completed: file=%s cracked=%s", file_id, job.cracked)
+
+    except Exception as exc:
+        logger.exception("Enrichment failed for file=%s: %s", file_id, exc)
+        if job is not None:
+            job.state = 'error'
+            job.message = str(exc)[:500]
+            try:
+                job.save()
+            except Exception:
+                pass
+    finally:
+        # Close only the worker thread's connection, never the request connection.
+        if close_conn:
+            connection.close()