build 4.6 Demo
This commit is contained in:
DaStra3003
@@ -0,0 +1,168 @@
|
||||
"""Optional hash cracking and endpoint probing with progress persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from django.db import connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def start_deep_analysis(file_id: int, url_ids=None, hash_ids=None,
|
||||
synchronous: bool = False) -> None:
|
||||
"""Start enrichment for all artifacts or the explicitly selected raw IDs."""
|
||||
if synchronous:
|
||||
_run(file_id, url_ids, hash_ids, close_conn=False)
|
||||
return
|
||||
thread = threading.Thread(target=_run, args=(file_id, url_ids, hash_ids), daemon=True,
|
||||
name=f"deep-analysis-{file_id}")
|
||||
thread.start()
|
||||
|
||||
|
||||
def _analyzed_dataset_for(raw_obj):
|
||||
"""Return the single analyzed dataset associated with a capture."""
|
||||
from IoTAnalyzer.models import AnalyzedDataset
|
||||
file_obj = raw_obj.raw_dataset.file
|
||||
ds, _ = AnalyzedDataset.objects.get_or_create(file=file_obj)
|
||||
return ds
|
||||
|
||||
|
||||
def crack_raw_hash(raw_hash) -> None:
|
||||
"""Crack one raw hash and persist the result idempotently."""
|
||||
from django.db import transaction
|
||||
from IoTAnalyzer.models import Tool, HashType, HashAnalyzed
|
||||
from analyzer.artifacts import HashArtifact
|
||||
from analyzer.enrich import crack_hash
|
||||
|
||||
try:
|
||||
res = crack_hash(HashArtifact(value=raw_hash.hash_value))
|
||||
cracked = bool(res.cracked and res.plaintext)
|
||||
logger.info("Hash id=%s mode=%s state=%s cracked=%s error=%s",
|
||||
raw_hash.id, res.hashcat_mode, res.state, cracked, res.error)
|
||||
|
||||
ds = _analyzed_dataset_for(raw_hash)
|
||||
tool, _ = Tool.objects.get_or_create(tool_name="Hashcat", defaults={"version": "6.x.x"})
|
||||
htype, _ = HashType.objects.get_or_create(hash_type=(res.hash_format or "unknown"))
|
||||
crack_time = f"{res.duration_seconds}s" if res.duration_seconds is not None else None
|
||||
|
||||
with transaction.atomic():
|
||||
ha, created = HashAnalyzed.objects.get_or_create(
|
||||
analyzed_dataset=ds, hash_value=res.value,
|
||||
defaults={"tool": tool, "cracked": cracked,
|
||||
"cracked_value": res.plaintext if cracked else None,
|
||||
"crack_time": crack_time},
|
||||
)
|
||||
if not created:
|
||||
ha.tool = tool
|
||||
ha.cracked = cracked
|
||||
ha.cracked_value = res.plaintext if cracked else None
|
||||
ha.crack_time = crack_time
|
||||
ha.save()
|
||||
ha.hash_type.add(htype)
|
||||
logger.info("Stored hash result id=%s (created=%s, cracked=%s)", ha.id, created, cracked)
|
||||
except Exception as exc:
|
||||
logger.exception("Hash enrichment failed (id=%s): %s", raw_hash.id, exc)
|
||||
|
||||
|
||||
def probe_raw_url(raw_url) -> None:
|
||||
"""Probe one raw endpoint and persist DNS, port, and TLS results."""
|
||||
from django.db import transaction
|
||||
from IoTAnalyzer.models import Tool, UrlAnalyzed, Port, TlsVersion, CipherSuite
|
||||
from analyzer.enrich import probe_url
|
||||
|
||||
try:
|
||||
probe = probe_url(raw_url.url, active=True)
|
||||
logger.info("URL id=%s ports=%s tls=%s error=%s",
|
||||
raw_url.id, probe.open_ports, probe.tls_versions, probe.error)
|
||||
|
||||
ds = _analyzed_dataset_for(raw_url)
|
||||
tool, _ = Tool.objects.get_or_create(tool_name="DNS Analyzer", defaults={"version": "1.0"})
|
||||
|
||||
with transaction.atomic():
|
||||
ua, created = UrlAnalyzed.objects.get_or_create(
|
||||
analyzed_dataset=ds, url=raw_url.url, defaults={"tool": tool})
|
||||
for p in probe.open_ports:
|
||||
port, _ = Port.objects.get_or_create(port=p)
|
||||
ua.ports.add(port)
|
||||
for v in (probe.tls_versions or ["N/A"]):
|
||||
tls, _ = TlsVersion.objects.get_or_create(tls_version=v)
|
||||
ua.tls_versions.add(tls)
|
||||
for c in (probe.cipher_suites or ["N/A"]):
|
||||
cs, _ = CipherSuite.objects.get_or_create(cipher_suite=c)
|
||||
ua.cipher_suites.add(cs)
|
||||
logger.info("Stored endpoint result id=%s (created=%s)", ua.id, created)
|
||||
except Exception as exc:
|
||||
logger.exception("Endpoint enrichment failed (id=%s): %s", raw_url.id, exc)
|
||||
|
||||
|
||||
def _run(file_id: int, url_ids=None, hash_ids=None, close_conn: bool = True) -> None:
|
||||
from IoTAnalyzer.models import UrlRaw, HashRaw, HashAnalyzed, DeepAnalysisJob
|
||||
|
||||
job = None
|
||||
try:
|
||||
url_qs = UrlRaw.objects.filter(raw_dataset__file_id=file_id)
|
||||
hash_qs = HashRaw.objects.filter(raw_dataset__file_id=file_id)
|
||||
# None selects all records; an empty list intentionally selects none.
|
||||
if url_ids is not None:
|
||||
url_qs = url_qs.filter(id__in=url_ids)
|
||||
if hash_ids is not None:
|
||||
hash_qs = hash_qs.filter(id__in=hash_ids)
|
||||
urls = list(url_qs)
|
||||
hashes = list(hash_qs)
|
||||
|
||||
DeepAnalysisJob.objects.filter(file_id=file_id).delete()
|
||||
job = DeepAnalysisJob.objects.create(
|
||||
file_id=file_id, state='running',
|
||||
url_total=len(urls), hash_total=len(hashes),
|
||||
message='Enrichment started',
|
||||
)
|
||||
logger.info("Enrichment started: file=%s urls=%s hashes=%s",
|
||||
file_id, len(urls), len(hashes))
|
||||
|
||||
# Crack hashes first because they usually complete quickly.
|
||||
for raw_hash in hashes:
|
||||
try:
|
||||
crack_raw_hash(raw_hash)
|
||||
except Exception as exc:
|
||||
logger.exception("Hash cracking failed (id=%s): %s", raw_hash.id, exc)
|
||||
job.hash_done += 1
|
||||
job.message = f"Hash-Cracking {job.hash_done}/{job.hash_total}"
|
||||
job.save(update_fields=['hash_done', 'message', 'updated'])
|
||||
|
||||
# Endpoint probing may perform DNS, port, and TLS checks.
|
||||
for raw_url in urls:
|
||||
try:
|
||||
probe_raw_url(raw_url)
|
||||
except Exception as exc:
|
||||
logger.exception("Endpoint probing failed (id=%s): %s", raw_url.id, exc)
|
||||
job.url_done += 1
|
||||
job.message = f"Endpoint probing {job.url_done}/{job.url_total}"
|
||||
job.save(update_fields=['url_done', 'message', 'updated'])
|
||||
|
||||
try:
|
||||
job.cracked = HashAnalyzed.objects.filter(
|
||||
analyzed_dataset__file_id=file_id, cracked=True
|
||||
).count()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
job.state = 'done'
|
||||
job.message = 'Enrichment completed'
|
||||
job.save()
|
||||
logger.info("Enrichment completed: file=%s cracked=%s", file_id, job.cracked)
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception("Enrichment failed for file=%s: %s", file_id, exc)
|
||||
if job is not None:
|
||||
job.state = 'error'
|
||||
job.message = str(exc)[:500]
|
||||
try:
|
||||
job.save()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
# Close only the worker thread's connection, never the request connection.
|
||||
if close_conn:
|
||||
connection.close()
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Persist results and artifacts produced by the shared analysis context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from django.conf import settings
|
||||
from analyzer import AnalysisEngine, extract_urls, extract_hashes
|
||||
from analyzer.extract_pyshark import build_context_from_pcap
|
||||
from analyzer.findings import Severity
|
||||
from IoTAnalyzer.models import FindingReport, UrlRaw, HashRaw
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CANDIDATE_DIRS = [
|
||||
str(settings.CAPTURE_UPLOAD_DIR),
|
||||
]
|
||||
|
||||
|
||||
def resolve_capture_path(filename: str) -> str | None:
|
||||
"""Resolve a stored capture name against supported upload locations."""
|
||||
if not filename:
|
||||
return None
|
||||
if os.path.isabs(filename) and os.path.exists(filename):
|
||||
return filename
|
||||
for base in _CANDIDATE_DIRS:
|
||||
candidate = os.path.join(base, filename)
|
||||
if os.path.exists(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _max_severity(result) -> str:
|
||||
worst = Severity.INFO
|
||||
for f in result.findings:
|
||||
if f.severity.rank > worst.rank:
|
||||
worst = f.severity
|
||||
return worst.value
|
||||
|
||||
|
||||
def collect_cloud_results(file_id: int) -> dict:
|
||||
"""Return deduplicated capture artifacts and any enrichment results."""
|
||||
from IoTAnalyzer.models import UrlRaw, HashRaw, UrlAnalyzed, HashAnalyzed
|
||||
|
||||
# Retain raw IDs so the frontend can request a selected enrichment subset.
|
||||
raw_urls = list(UrlRaw.objects.filter(raw_dataset__file_id=file_id)
|
||||
.values_list('id', 'url'))
|
||||
raw_hashes = list(HashRaw.objects.filter(raw_dataset__file_id=file_id)
|
||||
.values_list('id', 'hash_value'))
|
||||
|
||||
url_map = {}
|
||||
for ua in (UrlAnalyzed.objects.filter(analyzed_dataset__file_id=file_id)
|
||||
.prefetch_related('ports', 'tls_versions', 'cipher_suites')):
|
||||
url_map[ua.url] = {
|
||||
"ports": sorted({p.port for p in ua.ports.all()}),
|
||||
"tls_versions": [t.tls_version for t in ua.tls_versions.all()],
|
||||
"cipher_suites": [c.cipher_suite for c in ua.cipher_suites.all()],
|
||||
}
|
||||
|
||||
hash_map = {}
|
||||
for ha in (HashAnalyzed.objects.filter(analyzed_dataset__file_id=file_id)
|
||||
.prefetch_related('hash_type')):
|
||||
entry = {
|
||||
"cracked": ha.cracked,
|
||||
"cracked_value": ha.cracked_value,
|
||||
"crack_time": ha.crack_time,
|
||||
"hash_type": [h.hash_type for h in ha.hash_type.all()],
|
||||
}
|
||||
existing = hash_map.get(ha.hash_value)
|
||||
if existing is None or (ha.cracked and not existing.get("cracked")):
|
||||
hash_map[ha.hash_value] = entry
|
||||
|
||||
urls, seen = [], set()
|
||||
for raw_id, u in raw_urls:
|
||||
if not u or u in seen:
|
||||
continue
|
||||
seen.add(u)
|
||||
a = url_map.get(u)
|
||||
urls.append({"id": raw_id, "url": u, "analyzed": a is not None, **(a or {})})
|
||||
|
||||
hashes, seen_h = [], set()
|
||||
for raw_id, h in raw_hashes:
|
||||
if not h or h in seen_h:
|
||||
continue
|
||||
seen_h.add(h)
|
||||
a = hash_map.get(h)
|
||||
hashes.append({"id": raw_id, "hash_value": h, "analyzed": a is not None,
|
||||
**(a or {"cracked": False, "cracked_value": None,
|
||||
"crack_time": None, "hash_type": []})})
|
||||
|
||||
return {"urls": urls, "hashes": hashes}
|
||||
|
||||
|
||||
def _persist_artifacts(ctx, raw_dataset_id: int) -> None:
|
||||
"""Persist URL and hash artifacts extracted during the analysis pass."""
|
||||
if raw_dataset_id is None:
|
||||
return
|
||||
try:
|
||||
for u in extract_urls(ctx):
|
||||
UrlRaw.objects.get_or_create(raw_dataset_id=raw_dataset_id, url=u.value[:200])
|
||||
except Exception as exc:
|
||||
logger.exception("URL persistence failed: %s", exc)
|
||||
try:
|
||||
for h in extract_hashes(ctx):
|
||||
HashRaw.objects.get_or_create(raw_dataset_id=raw_dataset_id, hash_value=h.value)
|
||||
except Exception as exc:
|
||||
logger.exception("Hash persistence failed: %s", exc)
|
||||
|
||||
|
||||
def analyze_capture(filename: str, file_id: int, raw_dataset_id: int | None = None):
|
||||
"""Analyze a capture once and persist findings plus extracted artifacts."""
|
||||
path = resolve_capture_path(filename)
|
||||
if not path:
|
||||
logger.warning("Capture file not found: %s", filename)
|
||||
return None
|
||||
|
||||
try:
|
||||
ctx = build_context_from_pcap(path)
|
||||
result = AnalysisEngine().analyze_context(ctx)
|
||||
except Exception as exc:
|
||||
logger.exception("Security analysis failed for %s: %s", filename, exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
# Reanalysis replaces the previous report atomically at the file level.
|
||||
FindingReport.objects.filter(file_id=file_id).delete()
|
||||
FindingReport.objects.create(
|
||||
file_id=file_id,
|
||||
result_json=json.dumps(result.to_dict()),
|
||||
finding_count=len(result.findings),
|
||||
max_severity=_max_severity(result),
|
||||
)
|
||||
logger.info("Stored finding report: file_id=%s, findings=%s",
|
||||
file_id, len(result.findings))
|
||||
except Exception as exc:
|
||||
logger.exception("Finding report persistence failed: %s", exc)
|
||||
|
||||
_persist_artifacts(ctx, raw_dataset_id)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def run_security_analysis(filename: str, file_id: int):
|
||||
return analyze_capture(filename, file_id, raw_dataset_id=None)
|
||||
Reference in New Issue
Block a user