169 lines
6.8 KiB
Python
169 lines
6.8 KiB
Python
"""Optional hash cracking and endpoint probing with progress persistence."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import threading
|
|
|
|
from django.db import connection
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def start_deep_analysis(file_id: int, url_ids=None, hash_ids=None,
|
|
synchronous: bool = False) -> None:
|
|
"""Start enrichment for all artifacts or the explicitly selected raw IDs."""
|
|
if synchronous:
|
|
_run(file_id, url_ids, hash_ids, close_conn=False)
|
|
return
|
|
thread = threading.Thread(target=_run, args=(file_id, url_ids, hash_ids), daemon=True,
|
|
name=f"deep-analysis-{file_id}")
|
|
thread.start()
|
|
|
|
|
|
def _analyzed_dataset_for(raw_obj):
|
|
"""Return the single analyzed dataset associated with a capture."""
|
|
from IoTAnalyzer.models import AnalyzedDataset
|
|
file_obj = raw_obj.raw_dataset.file
|
|
ds, _ = AnalyzedDataset.objects.get_or_create(file=file_obj)
|
|
return ds
|
|
|
|
|
|
def crack_raw_hash(raw_hash) -> None:
|
|
"""Crack one raw hash and persist the result idempotently."""
|
|
from django.db import transaction
|
|
from IoTAnalyzer.models import Tool, HashType, HashAnalyzed
|
|
from analyzer.artifacts import HashArtifact
|
|
from analyzer.enrich import crack_hash
|
|
|
|
try:
|
|
res = crack_hash(HashArtifact(value=raw_hash.hash_value))
|
|
cracked = bool(res.cracked and res.plaintext)
|
|
logger.info("Hash id=%s mode=%s state=%s cracked=%s error=%s",
|
|
raw_hash.id, res.hashcat_mode, res.state, cracked, res.error)
|
|
|
|
ds = _analyzed_dataset_for(raw_hash)
|
|
tool, _ = Tool.objects.get_or_create(tool_name="Hashcat", defaults={"version": "6.x.x"})
|
|
htype, _ = HashType.objects.get_or_create(hash_type=(res.hash_format or "unknown"))
|
|
crack_time = f"{res.duration_seconds}s" if res.duration_seconds is not None else None
|
|
|
|
with transaction.atomic():
|
|
ha, created = HashAnalyzed.objects.get_or_create(
|
|
analyzed_dataset=ds, hash_value=res.value,
|
|
defaults={"tool": tool, "cracked": cracked,
|
|
"cracked_value": res.plaintext if cracked else None,
|
|
"crack_time": crack_time},
|
|
)
|
|
if not created:
|
|
ha.tool = tool
|
|
ha.cracked = cracked
|
|
ha.cracked_value = res.plaintext if cracked else None
|
|
ha.crack_time = crack_time
|
|
ha.save()
|
|
ha.hash_type.add(htype)
|
|
logger.info("Stored hash result id=%s (created=%s, cracked=%s)", ha.id, created, cracked)
|
|
except Exception as exc:
|
|
logger.exception("Hash enrichment failed (id=%s): %s", raw_hash.id, exc)
|
|
|
|
|
|
def probe_raw_url(raw_url) -> None:
|
|
"""Probe one raw endpoint and persist DNS, port, and TLS results."""
|
|
from django.db import transaction
|
|
from IoTAnalyzer.models import Tool, UrlAnalyzed, Port, TlsVersion, CipherSuite
|
|
from analyzer.enrich import probe_url
|
|
|
|
try:
|
|
probe = probe_url(raw_url.url, active=True)
|
|
logger.info("URL id=%s ports=%s tls=%s error=%s",
|
|
raw_url.id, probe.open_ports, probe.tls_versions, probe.error)
|
|
|
|
ds = _analyzed_dataset_for(raw_url)
|
|
tool, _ = Tool.objects.get_or_create(tool_name="DNS Analyzer", defaults={"version": "1.0"})
|
|
|
|
with transaction.atomic():
|
|
ua, created = UrlAnalyzed.objects.get_or_create(
|
|
analyzed_dataset=ds, url=raw_url.url, defaults={"tool": tool})
|
|
for p in probe.open_ports:
|
|
port, _ = Port.objects.get_or_create(port=p)
|
|
ua.ports.add(port)
|
|
for v in (probe.tls_versions or ["N/A"]):
|
|
tls, _ = TlsVersion.objects.get_or_create(tls_version=v)
|
|
ua.tls_versions.add(tls)
|
|
for c in (probe.cipher_suites or ["N/A"]):
|
|
cs, _ = CipherSuite.objects.get_or_create(cipher_suite=c)
|
|
ua.cipher_suites.add(cs)
|
|
logger.info("Stored endpoint result id=%s (created=%s)", ua.id, created)
|
|
except Exception as exc:
|
|
logger.exception("Endpoint enrichment failed (id=%s): %s", raw_url.id, exc)
|
|
|
|
|
|
def _run(file_id: int, url_ids=None, hash_ids=None, close_conn: bool = True) -> None:
|
|
from IoTAnalyzer.models import UrlRaw, HashRaw, HashAnalyzed, DeepAnalysisJob
|
|
|
|
job = None
|
|
try:
|
|
url_qs = UrlRaw.objects.filter(raw_dataset__file_id=file_id)
|
|
hash_qs = HashRaw.objects.filter(raw_dataset__file_id=file_id)
|
|
# None selects all records; an empty list intentionally selects none.
|
|
if url_ids is not None:
|
|
url_qs = url_qs.filter(id__in=url_ids)
|
|
if hash_ids is not None:
|
|
hash_qs = hash_qs.filter(id__in=hash_ids)
|
|
urls = list(url_qs)
|
|
hashes = list(hash_qs)
|
|
|
|
DeepAnalysisJob.objects.filter(file_id=file_id).delete()
|
|
job = DeepAnalysisJob.objects.create(
|
|
file_id=file_id, state='running',
|
|
url_total=len(urls), hash_total=len(hashes),
|
|
message='Enrichment started',
|
|
)
|
|
logger.info("Enrichment started: file=%s urls=%s hashes=%s",
|
|
file_id, len(urls), len(hashes))
|
|
|
|
# Crack hashes first because they usually complete quickly.
|
|
for raw_hash in hashes:
|
|
try:
|
|
crack_raw_hash(raw_hash)
|
|
except Exception as exc:
|
|
logger.exception("Hash cracking failed (id=%s): %s", raw_hash.id, exc)
|
|
job.hash_done += 1
|
|
job.message = f"Hash-Cracking {job.hash_done}/{job.hash_total}"
|
|
job.save(update_fields=['hash_done', 'message', 'updated'])
|
|
|
|
# Endpoint probing may perform DNS, port, and TLS checks.
|
|
for raw_url in urls:
|
|
try:
|
|
probe_raw_url(raw_url)
|
|
except Exception as exc:
|
|
logger.exception("Endpoint probing failed (id=%s): %s", raw_url.id, exc)
|
|
job.url_done += 1
|
|
job.message = f"Endpoint probing {job.url_done}/{job.url_total}"
|
|
job.save(update_fields=['url_done', 'message', 'updated'])
|
|
|
|
try:
|
|
job.cracked = HashAnalyzed.objects.filter(
|
|
analyzed_dataset__file_id=file_id, cracked=True
|
|
).count()
|
|
except Exception:
|
|
pass
|
|
|
|
job.state = 'done'
|
|
job.message = 'Enrichment completed'
|
|
job.save()
|
|
logger.info("Enrichment completed: file=%s cracked=%s", file_id, job.cracked)
|
|
|
|
except Exception as exc:
|
|
logger.exception("Enrichment failed for file=%s: %s", file_id, exc)
|
|
if job is not None:
|
|
job.state = 'error'
|
|
job.message = str(exc)[:500]
|
|
try:
|
|
job.save()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
# Close only the worker thread's connection, never the request connection.
|
|
if close_conn:
|
|
connection.close()
|