"""Optional hash cracking and endpoint probing with progress persistence.""" from __future__ import annotations import logging import threading from django.db import connection logger = logging.getLogger(__name__) def start_deep_analysis(file_id: int, url_ids=None, hash_ids=None, synchronous: bool = False) -> None: """Start enrichment for all artifacts or the explicitly selected raw IDs.""" if synchronous: _run(file_id, url_ids, hash_ids, close_conn=False) return thread = threading.Thread(target=_run, args=(file_id, url_ids, hash_ids), daemon=True, name=f"deep-analysis-{file_id}") thread.start() def _analyzed_dataset_for(raw_obj): """Return the single analyzed dataset associated with a capture.""" from IoTAnalyzer.models import AnalyzedDataset file_obj = raw_obj.raw_dataset.file ds, _ = AnalyzedDataset.objects.get_or_create(file=file_obj) return ds def crack_raw_hash(raw_hash) -> None: """Crack one raw hash and persist the result idempotently.""" from django.db import transaction from IoTAnalyzer.models import Tool, HashType, HashAnalyzed from analyzer.artifacts import HashArtifact from analyzer.enrich import crack_hash try: res = crack_hash(HashArtifact(value=raw_hash.hash_value)) cracked = bool(res.cracked and res.plaintext) logger.info("Hash id=%s mode=%s state=%s cracked=%s error=%s", raw_hash.id, res.hashcat_mode, res.state, cracked, res.error) ds = _analyzed_dataset_for(raw_hash) tool, _ = Tool.objects.get_or_create(tool_name="Hashcat", defaults={"version": "6.x.x"}) htype, _ = HashType.objects.get_or_create(hash_type=(res.hash_format or "unknown")) crack_time = f"{res.duration_seconds}s" if res.duration_seconds is not None else None with transaction.atomic(): ha, created = HashAnalyzed.objects.get_or_create( analyzed_dataset=ds, hash_value=res.value, defaults={"tool": tool, "cracked": cracked, "cracked_value": res.plaintext if cracked else None, "crack_time": crack_time}, ) if not created: ha.tool = tool ha.cracked = cracked ha.cracked_value = res.plaintext if cracked else None ha.crack_time = crack_time ha.save() ha.hash_type.add(htype) logger.info("Stored hash result id=%s (created=%s, cracked=%s)", ha.id, created, cracked) except Exception as exc: logger.exception("Hash enrichment failed (id=%s): %s", raw_hash.id, exc) def probe_raw_url(raw_url) -> None: """Probe one raw endpoint and persist DNS, port, and TLS results.""" from django.db import transaction from IoTAnalyzer.models import Tool, UrlAnalyzed, Port, TlsVersion, CipherSuite from analyzer.enrich import probe_url try: probe = probe_url(raw_url.url, active=True) logger.info("URL id=%s ports=%s tls=%s error=%s", raw_url.id, probe.open_ports, probe.tls_versions, probe.error) ds = _analyzed_dataset_for(raw_url) tool, _ = Tool.objects.get_or_create(tool_name="DNS Analyzer", defaults={"version": "1.0"}) with transaction.atomic(): ua, created = UrlAnalyzed.objects.get_or_create( analyzed_dataset=ds, url=raw_url.url, defaults={"tool": tool}) for p in probe.open_ports: port, _ = Port.objects.get_or_create(port=p) ua.ports.add(port) for v in (probe.tls_versions or ["N/A"]): tls, _ = TlsVersion.objects.get_or_create(tls_version=v) ua.tls_versions.add(tls) for c in (probe.cipher_suites or ["N/A"]): cs, _ = CipherSuite.objects.get_or_create(cipher_suite=c) ua.cipher_suites.add(cs) logger.info("Stored endpoint result id=%s (created=%s)", ua.id, created) except Exception as exc: logger.exception("Endpoint enrichment failed (id=%s): %s", raw_url.id, exc) def _run(file_id: int, url_ids=None, hash_ids=None, close_conn: bool = True) -> None: from IoTAnalyzer.models import UrlRaw, HashRaw, HashAnalyzed, DeepAnalysisJob job = None try: url_qs = UrlRaw.objects.filter(raw_dataset__file_id=file_id) hash_qs = HashRaw.objects.filter(raw_dataset__file_id=file_id) # None selects all records; an empty list intentionally selects none. if url_ids is not None: url_qs = url_qs.filter(id__in=url_ids) if hash_ids is not None: hash_qs = hash_qs.filter(id__in=hash_ids) urls = list(url_qs) hashes = list(hash_qs) DeepAnalysisJob.objects.filter(file_id=file_id).delete() job = DeepAnalysisJob.objects.create( file_id=file_id, state='running', url_total=len(urls), hash_total=len(hashes), message='Enrichment started', ) logger.info("Enrichment started: file=%s urls=%s hashes=%s", file_id, len(urls), len(hashes)) # Crack hashes first because they usually complete quickly. for raw_hash in hashes: try: crack_raw_hash(raw_hash) except Exception as exc: logger.exception("Hash cracking failed (id=%s): %s", raw_hash.id, exc) job.hash_done += 1 job.message = f"Hash-Cracking {job.hash_done}/{job.hash_total}" job.save(update_fields=['hash_done', 'message', 'updated']) # Endpoint probing may perform DNS, port, and TLS checks. for raw_url in urls: try: probe_raw_url(raw_url) except Exception as exc: logger.exception("Endpoint probing failed (id=%s): %s", raw_url.id, exc) job.url_done += 1 job.message = f"Endpoint probing {job.url_done}/{job.url_total}" job.save(update_fields=['url_done', 'message', 'updated']) try: job.cracked = HashAnalyzed.objects.filter( analyzed_dataset__file_id=file_id, cracked=True ).count() except Exception: pass job.state = 'done' job.message = 'Enrichment completed' job.save() logger.info("Enrichment completed: file=%s cracked=%s", file_id, job.cracked) except Exception as exc: logger.exception("Enrichment failed for file=%s: %s", file_id, exc) if job is not None: job.state = 'error' job.message = str(exc)[:500] try: job.save() except Exception: pass finally: # Close only the worker thread's connection, never the request connection. if close_conn: connection.close()