build 4.6 Demo

This commit is contained in:
DaStra3003
2026-08-19 08:04:59 +02:00
commit ff8f1ca989
123 changed files with 17696 additions and 0 deletions
+51
View File
@@ -0,0 +1,51 @@
"""Django admin registrations for demo operations."""
from django.contrib import admin
from .models import (
AnalyzedDataset,
CipherSuite,
DeepAnalysisJob,
Device,
DeviceType,
File,
FindingReport,
Firmware,
FirmwareDevices,
HashAnalyzed,
HashRaw,
HashType,
Manufacturer,
Port,
Product,
RawDataset,
TlsVersion,
Tool,
UrlAnalyzed,
UrlRaw,
)
admin.site.register(
[
Manufacturer,
DeviceType,
Product,
Device,
Firmware,
FirmwareDevices,
File,
RawDataset,
HashRaw,
UrlRaw,
Tool,
Port,
TlsVersion,
CipherSuite,
HashType,
AnalyzedDataset,
UrlAnalyzed,
HashAnalyzed,
FindingReport,
DeepAnalysisJob,
]
)
@@ -0,0 +1,168 @@
"""Optional hash cracking and endpoint probing with progress persistence."""
from __future__ import annotations
import logging
import threading
from django.db import connection
logger = logging.getLogger(__name__)
def start_deep_analysis(file_id: int, url_ids=None, hash_ids=None,
synchronous: bool = False) -> None:
"""Start enrichment for all artifacts or the explicitly selected raw IDs."""
if synchronous:
_run(file_id, url_ids, hash_ids, close_conn=False)
return
thread = threading.Thread(target=_run, args=(file_id, url_ids, hash_ids), daemon=True,
name=f"deep-analysis-{file_id}")
thread.start()
def _analyzed_dataset_for(raw_obj):
"""Return the single analyzed dataset associated with a capture."""
from IoTAnalyzer.models import AnalyzedDataset
file_obj = raw_obj.raw_dataset.file
ds, _ = AnalyzedDataset.objects.get_or_create(file=file_obj)
return ds
def crack_raw_hash(raw_hash) -> None:
"""Crack one raw hash and persist the result idempotently."""
from django.db import transaction
from IoTAnalyzer.models import Tool, HashType, HashAnalyzed
from analyzer.artifacts import HashArtifact
from analyzer.enrich import crack_hash
try:
res = crack_hash(HashArtifact(value=raw_hash.hash_value))
cracked = bool(res.cracked and res.plaintext)
logger.info("Hash id=%s mode=%s state=%s cracked=%s error=%s",
raw_hash.id, res.hashcat_mode, res.state, cracked, res.error)
ds = _analyzed_dataset_for(raw_hash)
tool, _ = Tool.objects.get_or_create(tool_name="Hashcat", defaults={"version": "6.x.x"})
htype, _ = HashType.objects.get_or_create(hash_type=(res.hash_format or "unknown"))
crack_time = f"{res.duration_seconds}s" if res.duration_seconds is not None else None
with transaction.atomic():
ha, created = HashAnalyzed.objects.get_or_create(
analyzed_dataset=ds, hash_value=res.value,
defaults={"tool": tool, "cracked": cracked,
"cracked_value": res.plaintext if cracked else None,
"crack_time": crack_time},
)
if not created:
ha.tool = tool
ha.cracked = cracked
ha.cracked_value = res.plaintext if cracked else None
ha.crack_time = crack_time
ha.save()
ha.hash_type.add(htype)
logger.info("Stored hash result id=%s (created=%s, cracked=%s)", ha.id, created, cracked)
except Exception as exc:
logger.exception("Hash enrichment failed (id=%s): %s", raw_hash.id, exc)
def probe_raw_url(raw_url) -> None:
"""Probe one raw endpoint and persist DNS, port, and TLS results."""
from django.db import transaction
from IoTAnalyzer.models import Tool, UrlAnalyzed, Port, TlsVersion, CipherSuite
from analyzer.enrich import probe_url
try:
probe = probe_url(raw_url.url, active=True)
logger.info("URL id=%s ports=%s tls=%s error=%s",
raw_url.id, probe.open_ports, probe.tls_versions, probe.error)
ds = _analyzed_dataset_for(raw_url)
tool, _ = Tool.objects.get_or_create(tool_name="DNS Analyzer", defaults={"version": "1.0"})
with transaction.atomic():
ua, created = UrlAnalyzed.objects.get_or_create(
analyzed_dataset=ds, url=raw_url.url, defaults={"tool": tool})
for p in probe.open_ports:
port, _ = Port.objects.get_or_create(port=p)
ua.ports.add(port)
for v in (probe.tls_versions or ["N/A"]):
tls, _ = TlsVersion.objects.get_or_create(tls_version=v)
ua.tls_versions.add(tls)
for c in (probe.cipher_suites or ["N/A"]):
cs, _ = CipherSuite.objects.get_or_create(cipher_suite=c)
ua.cipher_suites.add(cs)
logger.info("Stored endpoint result id=%s (created=%s)", ua.id, created)
except Exception as exc:
logger.exception("Endpoint enrichment failed (id=%s): %s", raw_url.id, exc)
def _run(file_id: int, url_ids=None, hash_ids=None, close_conn: bool = True) -> None:
from IoTAnalyzer.models import UrlRaw, HashRaw, HashAnalyzed, DeepAnalysisJob
job = None
try:
url_qs = UrlRaw.objects.filter(raw_dataset__file_id=file_id)
hash_qs = HashRaw.objects.filter(raw_dataset__file_id=file_id)
# None selects all records; an empty list intentionally selects none.
if url_ids is not None:
url_qs = url_qs.filter(id__in=url_ids)
if hash_ids is not None:
hash_qs = hash_qs.filter(id__in=hash_ids)
urls = list(url_qs)
hashes = list(hash_qs)
DeepAnalysisJob.objects.filter(file_id=file_id).delete()
job = DeepAnalysisJob.objects.create(
file_id=file_id, state='running',
url_total=len(urls), hash_total=len(hashes),
message='Enrichment started',
)
logger.info("Enrichment started: file=%s urls=%s hashes=%s",
file_id, len(urls), len(hashes))
# Crack hashes first because they usually complete quickly.
for raw_hash in hashes:
try:
crack_raw_hash(raw_hash)
except Exception as exc:
logger.exception("Hash cracking failed (id=%s): %s", raw_hash.id, exc)
job.hash_done += 1
job.message = f"Hash-Cracking {job.hash_done}/{job.hash_total}"
job.save(update_fields=['hash_done', 'message', 'updated'])
# Endpoint probing may perform DNS, port, and TLS checks.
for raw_url in urls:
try:
probe_raw_url(raw_url)
except Exception as exc:
logger.exception("Endpoint probing failed (id=%s): %s", raw_url.id, exc)
job.url_done += 1
job.message = f"Endpoint probing {job.url_done}/{job.url_total}"
job.save(update_fields=['url_done', 'message', 'updated'])
try:
job.cracked = HashAnalyzed.objects.filter(
analyzed_dataset__file_id=file_id, cracked=True
).count()
except Exception:
pass
job.state = 'done'
job.message = 'Enrichment completed'
job.save()
logger.info("Enrichment completed: file=%s cracked=%s", file_id, job.cracked)
except Exception as exc:
logger.exception("Enrichment failed for file=%s: %s", file_id, exc)
if job is not None:
job.state = 'error'
job.message = str(exc)[:500]
try:
job.save()
except Exception:
pass
finally:
# Close only the worker thread's connection, never the request connection.
if close_conn:
connection.close()
@@ -0,0 +1,146 @@
"""Persist results and artifacts produced by the shared analysis context."""
from __future__ import annotations
import json
import logging
import os
from django.conf import settings
from analyzer import AnalysisEngine, extract_urls, extract_hashes
from analyzer.extract_pyshark import build_context_from_pcap
from analyzer.findings import Severity
from IoTAnalyzer.models import FindingReport, UrlRaw, HashRaw
logger = logging.getLogger(__name__)
_CANDIDATE_DIRS = [
str(settings.CAPTURE_UPLOAD_DIR),
]
def resolve_capture_path(filename: str) -> str | None:
"""Resolve a stored capture name against supported upload locations."""
if not filename:
return None
if os.path.isabs(filename) and os.path.exists(filename):
return filename
for base in _CANDIDATE_DIRS:
candidate = os.path.join(base, filename)
if os.path.exists(candidate):
return candidate
return None
def _max_severity(result) -> str:
worst = Severity.INFO
for f in result.findings:
if f.severity.rank > worst.rank:
worst = f.severity
return worst.value
def collect_cloud_results(file_id: int) -> dict:
"""Return deduplicated capture artifacts and any enrichment results."""
from IoTAnalyzer.models import UrlRaw, HashRaw, UrlAnalyzed, HashAnalyzed
# Retain raw IDs so the frontend can request a selected enrichment subset.
raw_urls = list(UrlRaw.objects.filter(raw_dataset__file_id=file_id)
.values_list('id', 'url'))
raw_hashes = list(HashRaw.objects.filter(raw_dataset__file_id=file_id)
.values_list('id', 'hash_value'))
url_map = {}
for ua in (UrlAnalyzed.objects.filter(analyzed_dataset__file_id=file_id)
.prefetch_related('ports', 'tls_versions', 'cipher_suites')):
url_map[ua.url] = {
"ports": sorted({p.port for p in ua.ports.all()}),
"tls_versions": [t.tls_version for t in ua.tls_versions.all()],
"cipher_suites": [c.cipher_suite for c in ua.cipher_suites.all()],
}
hash_map = {}
for ha in (HashAnalyzed.objects.filter(analyzed_dataset__file_id=file_id)
.prefetch_related('hash_type')):
entry = {
"cracked": ha.cracked,
"cracked_value": ha.cracked_value,
"crack_time": ha.crack_time,
"hash_type": [h.hash_type for h in ha.hash_type.all()],
}
existing = hash_map.get(ha.hash_value)
if existing is None or (ha.cracked and not existing.get("cracked")):
hash_map[ha.hash_value] = entry
urls, seen = [], set()
for raw_id, u in raw_urls:
if not u or u in seen:
continue
seen.add(u)
a = url_map.get(u)
urls.append({"id": raw_id, "url": u, "analyzed": a is not None, **(a or {})})
hashes, seen_h = [], set()
for raw_id, h in raw_hashes:
if not h or h in seen_h:
continue
seen_h.add(h)
a = hash_map.get(h)
hashes.append({"id": raw_id, "hash_value": h, "analyzed": a is not None,
**(a or {"cracked": False, "cracked_value": None,
"crack_time": None, "hash_type": []})})
return {"urls": urls, "hashes": hashes}
def _persist_artifacts(ctx, raw_dataset_id: int) -> None:
"""Persist URL and hash artifacts extracted during the analysis pass."""
if raw_dataset_id is None:
return
try:
for u in extract_urls(ctx):
UrlRaw.objects.get_or_create(raw_dataset_id=raw_dataset_id, url=u.value[:200])
except Exception as exc:
logger.exception("URL persistence failed: %s", exc)
try:
for h in extract_hashes(ctx):
HashRaw.objects.get_or_create(raw_dataset_id=raw_dataset_id, hash_value=h.value)
except Exception as exc:
logger.exception("Hash persistence failed: %s", exc)
def analyze_capture(filename: str, file_id: int, raw_dataset_id: int | None = None):
"""Analyze a capture once and persist findings plus extracted artifacts."""
path = resolve_capture_path(filename)
if not path:
logger.warning("Capture file not found: %s", filename)
return None
try:
ctx = build_context_from_pcap(path)
result = AnalysisEngine().analyze_context(ctx)
except Exception as exc:
logger.exception("Security analysis failed for %s: %s", filename, exc)
return None
try:
# Reanalysis replaces the previous report atomically at the file level.
FindingReport.objects.filter(file_id=file_id).delete()
FindingReport.objects.create(
file_id=file_id,
result_json=json.dumps(result.to_dict()),
finding_count=len(result.findings),
max_severity=_max_severity(result),
)
logger.info("Stored finding report: file_id=%s, findings=%s",
file_id, len(result.findings))
except Exception as exc:
logger.exception("Finding report persistence failed: %s", exc)
_persist_artifacts(ctx, raw_dataset_id)
return result
def run_security_analysis(filename: str, file_id: int):
return analyze_capture(filename, file_id, raw_dataset_id=None)
+6
View File
@@ -0,0 +1,6 @@
from django.apps import AppConfig
class IotanalyzerConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'IoTAnalyzer'
@@ -0,0 +1,13 @@
"""Session authentication adapted to the localhost-only demo API.
State-changing requests remain protected by authenticated session cookies and a
strict CORS allow-list. The separate CSRF check is disabled because the frontend
and API use different localhost ports.
"""
from rest_framework.authentication import SessionAuthentication
class DemoSessionAuthentication(SessionAuthentication):
def enforce_csrf(self, request):
return None
+122
View File
@@ -0,0 +1,122 @@
"""Focused creation services used by the demo API."""
import ipaddress
import logging
import os
from django.core.exceptions import ValidationError
from django.db import transaction
from .models import (
Device,
DeviceType,
File,
Firmware,
FirmwareDevices,
Manufacturer,
Product,
RawDataset,
)
logger = logging.getLogger(__name__)
CAPTURE_EXTENSIONS = {".pcap", ".pcapng"}
def _required_text(value, label: str, max_length: int) -> str:
if not isinstance(value, str) or not value.strip():
raise ValidationError(f"{label} is required.")
value = value.strip()
if len(value) > max_length:
raise ValidationError(f"{label} must not exceed {max_length} characters.")
return value
@transaction.atomic
def create_manufacturer(name: str) -> Manufacturer:
name = _required_text(name, "Manufacturer name", 50)
manufacturer, created = Manufacturer.objects.get_or_create(name=name)
if not created:
raise ValidationError("This manufacturer already exists.")
return manufacturer
@transaction.atomic
def create_device_type(device_type: str) -> DeviceType:
device_type = _required_text(device_type, "Device type", 50)
instance, created = DeviceType.objects.get_or_create(device_type=device_type)
if not created:
raise ValidationError("This device type already exists.")
return instance
@transaction.atomic
def create_new_device(
manufacturer_id: int,
device_type_id: int,
device_name: str,
serial: str,
comment: str = "",
) -> Device:
manufacturer = Manufacturer.objects.get(id=manufacturer_id)
device_type = DeviceType.objects.get(id=device_type_id)
device_name = _required_text(device_name, "Device name", 100)
serial = _required_text(serial, "Serial number", 100)
comment = (comment or "").strip()[:255]
product, _ = Product.objects.get_or_create(
manufacturer=manufacturer,
device_type=device_type,
product_name=device_name,
)
device, created = Device.objects.get_or_create(
product=product,
serial=serial,
defaults={"comment": comment},
)
if not created:
raise ValidationError("A device with this product and serial number already exists.")
return device
def create_capture(
device_id: int,
firmware_version: str,
filename: str,
ip_address: str,
) -> File:
"""Create capture metadata, analyze the stored file, and update its status."""
firmware_version = _required_text(firmware_version, "Firmware version", 100)
filename = os.path.basename(_required_text(filename, "Capture filename", 255))
if os.path.splitext(filename)[1].lower() not in CAPTURE_EXTENSIONS:
raise ValidationError("Only .pcap and .pcapng captures are supported.")
try:
normalized_ip = str(ipaddress.ip_address(ip_address))
except ValueError as exc:
raise ValidationError("A valid device IP address is required.") from exc
with transaction.atomic():
device = Device.objects.select_related("product").get(id=device_id)
firmware, _ = Firmware.objects.get_or_create(
product=device.product,
version=firmware_version,
)
FirmwareDevices.objects.get_or_create(device=device, firmware=firmware)
file_entry, created = File.objects.get_or_create(
device=device,
firmware=firmware,
filename=filename,
ip_address=normalized_ip,
defaults={"status": "analyzing"},
)
if not created:
raise ValidationError("This capture is already registered for the device.")
raw_dataset = RawDataset.objects.create(file=file_entry)
from .analyze.security_analysis import analyze_capture
result = analyze_capture(filename, file_entry.id, raw_dataset.id)
file_entry.status = "analyzed" if result is not None else "analysis_failed"
file_entry.save(update_fields=["status"])
logger.info("Capture %s completed with status %s", file_entry.id, file_entry.status)
return file_entry
+51
View File
@@ -0,0 +1,51 @@
"""Read projections used by the demo API."""
from .models import Device, DeviceType, File, Manufacturer
def get_all_manufacturers() -> list[dict]:
return [item.to_json() for item in Manufacturer.objects.all()]
def get_all_device_types() -> list[dict]:
return [item.to_json() for item in DeviceType.objects.all()]
def get_all_devices() -> list[dict]:
devices = Device.objects.select_related(
"product", "product__manufacturer", "product__device_type"
)
return [
{
"id": device.id,
"manufacturer_name": device.product.manufacturer.name,
"device_type": device.product.device_type.device_type,
"device_name": device.product.product_name,
"serial": device.serial,
"comment": device.comment,
"created": device.created.isoformat(),
}
for device in devices
]
def get_device_by_id(device_id: int) -> dict:
device = Device.objects.select_related(
"product", "product__manufacturer", "product__device_type"
).get(id=device_id)
return device.to_json()
def get_files_by_device(device_id: int) -> list[dict]:
files = File.objects.filter(device_id=device_id).select_related("firmware")
return [
{
"id": file.id,
"filename": file.filename,
"ip": file.ip_address,
"state": file.status,
"upload": file.upload_time.isoformat(),
"firmware_version": file.firmware.version,
}
for file in files
]
@@ -0,0 +1,333 @@
# Generated by Django 4.2.20 on 2026-07-17 12:17
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='AnalyzedDataset',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_analyzed_dataset',
},
),
migrations.CreateModel(
name='CipherSuite',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('cipher_suite', models.CharField(max_length=100, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_cipher_suites',
},
),
migrations.CreateModel(
name='Device',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('serial', models.CharField(max_length=100)),
('comment', models.CharField(blank=True, default='', max_length=255)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_device',
},
),
migrations.CreateModel(
name='DeviceType',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('device_type', models.CharField(max_length=50, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_device_type',
'ordering': ['device_type'],
},
),
migrations.CreateModel(
name='File',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('filename', models.CharField(max_length=255)),
('ip_address', models.GenericIPAddressField()),
('status', models.CharField(default='pending', max_length=50)),
('upload_time', models.DateTimeField(auto_now_add=True)),
('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.device')),
],
options={
'db_table': 'iot_files',
'ordering': ['-upload_time'],
},
),
migrations.CreateModel(
name='Firmware',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('version', models.CharField(max_length=100)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_firmware',
},
),
migrations.CreateModel(
name='HashType',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('hash_type', models.CharField(max_length=100, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_hash_types',
},
),
migrations.CreateModel(
name='Manufacturer',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=50, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_manufacturer',
'ordering': ['name'],
},
),
migrations.CreateModel(
name='Port',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('port', models.PositiveIntegerField(unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_ports',
},
),
migrations.CreateModel(
name='RawDataset',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(auto_now_add=True)),
('file', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.file')),
],
options={
'db_table': 'iot_raw_dataset',
},
),
migrations.CreateModel(
name='TlsVersion',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('tls_version', models.CharField(max_length=50, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_tls_versions',
},
),
migrations.CreateModel(
name='Tool',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('tool_name', models.CharField(max_length=100, unique=True)),
('version', models.CharField(max_length=50)),
('created', models.DateTimeField(auto_now_add=True)),
],
options={
'db_table': 'iot_tool',
},
),
migrations.CreateModel(
name='UrlRaw',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.CharField(max_length=500)),
('created', models.DateTimeField(auto_now_add=True)),
('raw_dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.rawdataset')),
],
options={
'db_table': 'iot_raw_url',
},
),
migrations.CreateModel(
name='UrlAnalyzed',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.CharField(max_length=500)),
('created', models.DateTimeField(auto_now_add=True)),
('analyzed_dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.analyzeddataset')),
('cipher_suites', models.ManyToManyField(to='IoTAnalyzer.ciphersuite')),
('ports', models.ManyToManyField(to='IoTAnalyzer.port')),
('tls_versions', models.ManyToManyField(to='IoTAnalyzer.tlsversion')),
('tool', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.tool')),
],
options={
'db_table': 'iot_analyzed_url',
},
),
migrations.CreateModel(
name='Product',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('product_name', models.CharField(max_length=100)),
('created', models.DateTimeField(auto_now_add=True)),
('device_type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.devicetype')),
('manufacturer', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.manufacturer')),
],
options={
'db_table': 'iot_product',
'ordering': ['product_name'],
},
),
migrations.CreateModel(
name='HashRaw',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('hash_value', models.TextField()),
('created', models.DateTimeField(auto_now_add=True)),
('raw_dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.rawdataset')),
],
options={
'db_table': 'iot_raw_hash',
},
),
migrations.CreateModel(
name='HashAnalyzed',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('hash_value', models.TextField()),
('cracked', models.BooleanField(default=False)),
('cracked_value', models.CharField(blank=True, max_length=255, null=True)),
('crack_time', models.CharField(blank=True, max_length=50, null=True)),
('created', models.DateTimeField(auto_now_add=True)),
('analyzed_dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.analyzeddataset')),
('hash_type', models.ManyToManyField(to='IoTAnalyzer.hashtype')),
('tool', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.tool')),
],
options={
'db_table': 'iot_analyzed_hash',
},
),
migrations.CreateModel(
name='FirmwareDevices',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(auto_now_add=True)),
('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.device')),
('firmware', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.firmware')),
],
options={
'db_table': 'iot_firmware_devices',
},
),
migrations.AddField(
model_name='firmware',
name='devices',
field=models.ManyToManyField(through='IoTAnalyzer.FirmwareDevices', to='IoTAnalyzer.device'),
),
migrations.AddField(
model_name='firmware',
name='product',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.product'),
),
migrations.CreateModel(
name='FindingReport',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('result_json', models.TextField()),
('finding_count', models.IntegerField(default=0)),
('max_severity', models.CharField(default='info', max_length=20)),
('created', models.DateTimeField(auto_now_add=True)),
('file', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.file')),
],
options={
'db_table': 'iot_finding_report',
'ordering': ['-created'],
},
),
migrations.AddField(
model_name='file',
name='firmware',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.firmware'),
),
migrations.AddField(
model_name='device',
name='product',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='IoTAnalyzer.product'),
),
migrations.CreateModel(
name='DeepAnalysisJob',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('state', models.CharField(choices=[('pending', 'Pending'), ('running', 'Running'), ('done', 'Done'), ('error', 'Error')], default='pending', max_length=20)),
('url_total', models.IntegerField(default=0)),
('url_done', models.IntegerField(default=0)),
('hash_total', models.IntegerField(default=0)),
('hash_done', models.IntegerField(default=0)),
('cracked', models.IntegerField(default=0)),
('message', models.TextField(blank=True, default='')),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('file', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.file')),
],
options={
'db_table': 'iot_deep_analysis_job',
'ordering': ['-created'],
},
),
migrations.AddField(
model_name='analyzeddataset',
name='file',
field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='IoTAnalyzer.file'),
),
migrations.AddConstraint(
model_name='urlraw',
constraint=models.UniqueConstraint(fields=('raw_dataset', 'url'), name='unique_raw_url'),
),
migrations.AddConstraint(
model_name='urlanalyzed',
constraint=models.UniqueConstraint(fields=('analyzed_dataset', 'url'), name='unique_analyzed_url'),
),
migrations.AddConstraint(
model_name='product',
constraint=models.UniqueConstraint(fields=('manufacturer', 'device_type', 'product_name'), name='unique_demo_product'),
),
migrations.AddConstraint(
model_name='hashraw',
constraint=models.UniqueConstraint(fields=('raw_dataset', 'hash_value'), name='unique_raw_hash'),
),
migrations.AddConstraint(
model_name='hashanalyzed',
constraint=models.UniqueConstraint(fields=('analyzed_dataset', 'hash_value'), name='unique_analyzed_hash'),
),
migrations.AddConstraint(
model_name='firmwaredevices',
constraint=models.UniqueConstraint(fields=('firmware', 'device'), name='unique_device_firmware'),
),
migrations.AddConstraint(
model_name='firmware',
constraint=models.UniqueConstraint(fields=('product', 'version'), name='unique_product_firmware'),
),
migrations.AddConstraint(
model_name='file',
constraint=models.UniqueConstraint(fields=('device', 'firmware', 'filename', 'ip_address'), name='unique_device_capture'),
),
migrations.AddConstraint(
model_name='device',
constraint=models.UniqueConstraint(fields=('product', 'serial'), name='unique_demo_device'),
),
]
+337
View File
@@ -0,0 +1,337 @@
"""Persistence models for IoT PreTester 4.6 Demo."""
import json
from django.db import models
class Manufacturer(models.Model):
name = models.CharField(max_length=50, unique=True)
created = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {"id": self.id, "name": self.name, "created": self.created.isoformat()}
class Meta:
db_table = "iot_manufacturer"
ordering = ["name"]
class DeviceType(models.Model):
device_type = models.CharField(max_length=50, unique=True)
created = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {
"id": self.id,
"device_type": self.device_type,
"created": self.created.isoformat(),
}
class Meta:
db_table = "iot_device_type"
ordering = ["device_type"]
class Product(models.Model):
manufacturer = models.ForeignKey(Manufacturer, on_delete=models.PROTECT)
device_type = models.ForeignKey(DeviceType, on_delete=models.PROTECT)
product_name = models.CharField(max_length=100)
created = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {
"id": self.id,
"manufacturer": self.manufacturer.to_json(),
"product_name": self.product_name,
"device_type": self.device_type.to_json(),
"created": self.created.isoformat(),
}
class Meta:
db_table = "iot_product"
ordering = ["product_name"]
constraints = [
models.UniqueConstraint(
fields=["manufacturer", "device_type", "product_name"],
name="unique_demo_product",
)
]
class Device(models.Model):
product = models.ForeignKey(Product, on_delete=models.PROTECT)
serial = models.CharField(max_length=100)
comment = models.CharField(max_length=255, blank=True, default="")
created = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {
"id": self.id,
"product": self.product.to_json(),
"serial": self.serial,
"comment": self.comment,
"created": self.created.isoformat(),
}
class Meta:
db_table = "iot_device"
constraints = [
models.UniqueConstraint(
fields=["product", "serial"], name="unique_demo_device"
)
]
class Firmware(models.Model):
"""Firmware metadata associated with a capture; binaries are not analyzed."""
version = models.CharField(max_length=100)
product = models.ForeignKey(Product, on_delete=models.PROTECT)
devices = models.ManyToManyField(Device, through="FirmwareDevices")
created = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {
"id": self.id,
"version": self.version,
"product": self.product_id,
"created": self.created.isoformat(),
}
class Meta:
db_table = "iot_firmware"
constraints = [
models.UniqueConstraint(
fields=["product", "version"], name="unique_product_firmware"
)
]
class FirmwareDevices(models.Model):
firmware = models.ForeignKey(Firmware, on_delete=models.CASCADE)
device = models.ForeignKey(Device, on_delete=models.CASCADE)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_firmware_devices"
constraints = [
models.UniqueConstraint(
fields=["firmware", "device"], name="unique_device_firmware"
)
]
class File(models.Model):
firmware = models.ForeignKey(Firmware, on_delete=models.PROTECT)
device = models.ForeignKey(Device, on_delete=models.CASCADE)
filename = models.CharField(max_length=255)
ip_address = models.GenericIPAddressField()
status = models.CharField(max_length=50, default="pending")
upload_time = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {
"id": self.id,
"filename": self.filename,
"ip_address": self.ip_address,
"firmware_version": self.firmware.version,
"status": self.status,
"upload_time": self.upload_time.isoformat(),
}
class Meta:
db_table = "iot_files"
ordering = ["-upload_time"]
constraints = [
models.UniqueConstraint(
fields=["device", "firmware", "filename", "ip_address"],
name="unique_device_capture",
)
]
class RawDataset(models.Model):
file = models.OneToOneField(File, on_delete=models.CASCADE)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_raw_dataset"
class HashRaw(models.Model):
raw_dataset = models.ForeignKey(RawDataset, on_delete=models.CASCADE)
hash_value = models.TextField()
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_raw_hash"
constraints = [
models.UniqueConstraint(
fields=["raw_dataset", "hash_value"], name="unique_raw_hash"
)
]
class UrlRaw(models.Model):
raw_dataset = models.ForeignKey(RawDataset, on_delete=models.CASCADE)
url = models.CharField(max_length=500)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_raw_url"
constraints = [
models.UniqueConstraint(
fields=["raw_dataset", "url"], name="unique_raw_url"
)
]
class Tool(models.Model):
tool_name = models.CharField(max_length=100, unique=True)
version = models.CharField(max_length=50)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_tool"
class Port(models.Model):
port = models.PositiveIntegerField(unique=True)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_ports"
class TlsVersion(models.Model):
tls_version = models.CharField(max_length=50, unique=True)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_tls_versions"
class CipherSuite(models.Model):
cipher_suite = models.CharField(max_length=100, unique=True)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_cipher_suites"
class HashType(models.Model):
hash_type = models.CharField(max_length=100, unique=True)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_hash_types"
class AnalyzedDataset(models.Model):
file = models.OneToOneField(File, on_delete=models.CASCADE)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_analyzed_dataset"
class UrlAnalyzed(models.Model):
analyzed_dataset = models.ForeignKey(AnalyzedDataset, on_delete=models.CASCADE)
url = models.CharField(max_length=500)
tool = models.ForeignKey(Tool, on_delete=models.PROTECT)
ports = models.ManyToManyField(Port)
tls_versions = models.ManyToManyField(TlsVersion)
cipher_suites = models.ManyToManyField(CipherSuite)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_analyzed_url"
constraints = [
models.UniqueConstraint(
fields=["analyzed_dataset", "url"], name="unique_analyzed_url"
)
]
class HashAnalyzed(models.Model):
analyzed_dataset = models.ForeignKey(AnalyzedDataset, on_delete=models.CASCADE)
hash_type = models.ManyToManyField(HashType)
hash_value = models.TextField()
cracked = models.BooleanField(default=False)
cracked_value = models.CharField(max_length=255, blank=True, null=True)
crack_time = models.CharField(max_length=50, blank=True, null=True)
tool = models.ForeignKey(Tool, on_delete=models.PROTECT)
created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = "iot_analyzed_hash"
constraints = [
models.UniqueConstraint(
fields=["analyzed_dataset", "hash_value"], name="unique_analyzed_hash"
)
]
class FindingReport(models.Model):
file = models.OneToOneField(File, on_delete=models.CASCADE)
result_json = models.TextField()
finding_count = models.IntegerField(default=0)
max_severity = models.CharField(max_length=20, default="info")
created = models.DateTimeField(auto_now_add=True)
def to_json(self):
return {
"id": self.id,
"file_id": self.file_id,
"finding_count": self.finding_count,
"max_severity": self.max_severity,
"created": self.created.isoformat(),
"result": json.loads(self.result_json),
}
class Meta:
db_table = "iot_finding_report"
ordering = ["-created"]
class DeepAnalysisJob(models.Model):
STATE_CHOICES = [
("pending", "Pending"),
("running", "Running"),
("done", "Done"),
("error", "Error"),
]
file = models.OneToOneField(File, on_delete=models.CASCADE)
state = models.CharField(max_length=20, choices=STATE_CHOICES, default="pending")
url_total = models.IntegerField(default=0)
url_done = models.IntegerField(default=0)
hash_total = models.IntegerField(default=0)
hash_done = models.IntegerField(default=0)
cracked = models.IntegerField(default=0)
message = models.TextField(blank=True, default="")
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
def to_json(self):
done = self.url_done + self.hash_done
total = self.url_total + self.hash_total
return {
"file_id": self.file_id,
"state": self.state,
"url_total": self.url_total,
"url_done": self.url_done,
"hash_total": self.hash_total,
"hash_done": self.hash_done,
"cracked": self.cracked,
"progress_done": done,
"progress_total": total,
"percent": int(done * 100 / total) if total else (100 if self.state == "done" else 0),
"message": self.message,
"updated": self.updated.isoformat(),
}
class Meta:
db_table = "iot_deep_analysis_job"
ordering = ["-created"]
+32
View File
@@ -0,0 +1,32 @@
"""URL routes for the IoT PreTester 4.6 Demo API."""
from django.urls import path
from . import views
urlpatterns = [
path("api/health", views.HealthView.as_view(), name="health"),
path("api/login", views.LoginView.as_view(), name="login"),
path("api/logout", views.LogoutView.as_view(), name="logout"),
path("api/register", views.RegisterUserView.as_view(), name="register_user"),
path("api/list-users", views.ListUsersView.as_view(), name="list_users"),
path("api/delete-user", views.DeleteUserView.as_view(), name="delete_user"),
path("api/reset-password", views.ResetPasswordView.as_view(), name="reset_password"),
path("api/create_manufact", views.CreateManufacturerView.as_view(), name="create_manufacturer"),
path("api/create_device_type", views.CreateDeviceTypeView.as_view(), name="create_device_type"),
path("api/create_device", views.CreateDeviceView.as_view(), name="create_device"),
path("api/captures", views.CreateCaptureView.as_view(), name="create_capture"),
path("api/get_all_manufacturers", views.ManufacturersView.as_view(), name="manufacturers"),
path("api/get_all_devicetypes", views.DeviceTypesView.as_view(), name="device_types"),
path("api/get_all_devices", views.DevicesView.as_view(), name="devices"),
path("api/get_device_by_id/<int:device_id>", views.DeviceDetailView.as_view(), name="device_detail"),
path("api/get_file_by_device/<int:device_id>", views.DeviceCapturesView.as_view(), name="device_captures"),
path("api/delete_device/<int:device_id>", views.DeleteDeviceView.as_view(), name="delete_device"),
path("api/delete_manufacturer/<int:manufacturer_id>", views.DeleteManufacturerView.as_view(), name="delete_manufacturer"),
path("api/delete_device_type/<int:device_type_id>", views.DeleteDeviceTypeView.as_view(), name="delete_device_type"),
path("api/get_findings/<int:file_id>", views.FindingsView.as_view(), name="findings"),
path("api/get_cloud_results/<int:file_id>", views.CloudResultsView.as_view(), name="cloud_results"),
path("api/start_analysis/<int:file_id>", views.StartAnalysisView.as_view(), name="start_analysis"),
path("api/get_analysis_status/<int:file_id>", views.AnalysisStatusView.as_view(), name="analysis_status"),
path("api/get_report/<int:file_id>/<str:fmt>", views.ReportView.as_view(), name="report"),
]
+80
View File
@@ -0,0 +1,80 @@
"""User-account services for the controlled demo environment."""
from django.contrib.auth import authenticate, get_user_model, login, logout
from django.contrib.auth.password_validation import validate_password
from django.core.exceptions import ValidationError
from django.db import transaction
User = get_user_model()
@transaction.atomic
def register_user(username: str, password: str, role: str) -> dict:
username = (username or "").strip()
if not username:
raise ValidationError("Username is required.")
if role not in {"admin", "user"}:
raise ValidationError("Role must be admin or user.")
if User.objects.filter(username=username).exists():
raise ValidationError("This username already exists.")
validate_password(password)
is_admin = role == "admin"
User.objects.create_user(
username=username,
password=password,
is_superuser=is_admin,
is_staff=is_admin,
)
return {"status": "success", "message": "User created successfully."}
@transaction.atomic
def delete_user(request, user_id: int) -> dict:
if request.user.id == user_id:
raise ValidationError("You cannot delete your own account.")
user = User.objects.get(id=user_id)
user.delete()
return {"status": "success", "message": "User deleted successfully."}
@transaction.atomic
def set_password(user_id: int, new_password: str) -> dict:
user = User.objects.get(id=user_id)
validate_password(new_password, user)
user.set_password(new_password)
user.save(update_fields=["password"])
return {"status": "success", "message": "Password reset successfully."}
def login_user(request, username: str, password: str):
user = authenticate(request, username=(username or "").strip(), password=password)
if user is None:
return False, {"message": "Invalid username or password."}
login(request, user)
role = "admin" if user.is_superuser else "user"
return True, {
"message": "Login successful.",
"user": {"id": user.id, "username": user.username, "role": role},
}
def logout_user(request) -> dict:
logout(request)
return {"status": "success", "message": "Logout successful."}
def list_users() -> dict:
return {
"status": "success",
"data": [
{
"id": user.id,
"username": user.username,
"email": user.email,
"role": "admin" if user.is_superuser else "user",
}
for user in User.objects.order_by("username")
],
}
+382
View File
@@ -0,0 +1,382 @@
"""REST endpoints exposed by IoT PreTester 4.6 Demo."""
import json
import re
from pathlib import Path
from uuid import uuid4
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.core.files.storage import FileSystemStorage
from django.http import HttpResponse
from django.db.models.deletion import ProtectedError
from django.shortcuts import get_object_or_404
from rest_framework import serializers, status
from rest_framework.permissions import AllowAny, IsAdminUser
from rest_framework.response import Response
from rest_framework.views import APIView
from .analyze.security_analysis import collect_cloud_results
from .db_create import (
create_capture,
create_device_type,
create_manufacturer,
create_new_device,
)
from .db_read import (
get_all_device_types,
get_all_devices,
get_all_manufacturers,
get_device_by_id,
get_files_by_device,
)
from .models import (
DeepAnalysisJob,
Device,
DeviceType,
File,
FindingReport,
Manufacturer,
)
from .user_funcs import (
delete_user,
list_users,
login_user,
logout_user,
register_user,
set_password,
)
def _validation_error(exc: ValidationError) -> Response:
messages = getattr(exc, "messages", [str(exc)])
return Response({"error": messages[0]}, status=status.HTTP_400_BAD_REQUEST)
class HealthView(APIView):
permission_classes = [AllowAny]
def get(self, request):
return Response({"status": "ok", "service": "IoT PreTester 4.6 Demo"})
class LoginView(APIView):
permission_classes = [AllowAny]
def post(self, request):
username = request.data.get("username")
password = request.data.get("password")
if not username or not password:
return Response(
[False, {"message": "Username and password are required."}],
status=status.HTTP_400_BAD_REQUEST,
)
return Response(login_user(request, username, password))
class LogoutView(APIView):
def post(self, request):
return Response(logout_user(request))
class UserSerializer(serializers.Serializer):
username = serializers.CharField(max_length=150)
password = serializers.CharField(write_only=True)
role = serializers.ChoiceField(choices=["admin", "user"])
class RegisterUserView(APIView):
permission_classes = [IsAdminUser]
def post(self, request):
serializer = UserSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
try:
result = register_user(**serializer.validated_data)
except ValidationError as exc:
return _validation_error(exc)
return Response(result, status=status.HTTP_201_CREATED)
class ListUsersView(APIView):
permission_classes = [IsAdminUser]
def get(self, request):
return Response(list_users())
class DeleteUserView(APIView):
permission_classes = [IsAdminUser]
def post(self, request):
try:
user_id = int(request.data.get("id"))
return Response(delete_user(request, user_id))
except (TypeError, ValueError):
return Response({"error": "A valid user ID is required."}, status=400)
except ObjectDoesNotExist:
return Response({"error": "The user does not exist."}, status=404)
except ValidationError as exc:
return _validation_error(exc)
class ResetPasswordView(APIView):
permission_classes = [IsAdminUser]
def post(self, request):
user_id = request.data.get("user_id")
password = request.data.get("new_password")
if not isinstance(password, str) or not password:
return Response({"error": "A new password is required."}, status=400)
try:
return Response(set_password(int(user_id), password))
except (TypeError, ValueError):
return Response({"error": "A valid user ID is required."}, status=400)
except ObjectDoesNotExist:
return Response({"error": "The user does not exist."}, status=404)
except ValidationError as exc:
return _validation_error(exc)
class CreateManufacturerView(APIView):
permission_classes = [IsAdminUser]
def post(self, request):
try:
item = create_manufacturer(request.data.get("name"))
return Response(item.to_json(), status=status.HTTP_201_CREATED)
except ValidationError as exc:
return _validation_error(exc)
class CreateDeviceTypeView(APIView):
permission_classes = [IsAdminUser]
def post(self, request):
try:
item = create_device_type(request.data.get("device_type"))
return Response(item.to_json(), status=status.HTTP_201_CREATED)
except ValidationError as exc:
return _validation_error(exc)
class DeviceSerializer(serializers.Serializer):
device_type_id = serializers.IntegerField()
device_name = serializers.CharField(max_length=100)
serial = serializers.CharField(max_length=100)
manufacturer_id = serializers.IntegerField()
comment = serializers.CharField(max_length=255, required=False, allow_blank=True)
class CreateDeviceView(APIView):
permission_classes = [IsAdminUser]
def post(self, request):
serializer = DeviceSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
try:
device = create_new_device(**serializer.validated_data)
return Response(device.to_json(), status=status.HTTP_201_CREATED)
except ObjectDoesNotExist:
return Response({"error": "The manufacturer or device type does not exist."}, status=400)
except ValidationError as exc:
return _validation_error(exc)
class CaptureSerializer(serializers.Serializer):
capture = serializers.FileField(write_only=True)
device_id = serializers.IntegerField()
firmware_version = serializers.CharField(max_length=100)
ip_address = serializers.IPAddressField()
def validate_capture(self, capture):
if capture.size <= 0:
raise serializers.ValidationError("The capture is empty.")
if capture.size > 512 * 1024 * 1024:
raise serializers.ValidationError("The capture exceeds the 512 MB demo limit.")
if Path(capture.name).suffix.lower() not in {".pcap", ".pcapng"}:
raise serializers.ValidationError("Only PCAP and PCAPNG captures are supported.")
return capture
class CreateCaptureView(APIView):
def post(self, request):
serializer = CaptureSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
values = dict(serializer.validated_data)
uploaded_capture = values.pop("capture")
suffix = Path(uploaded_capture.name).suffix.lower()
stem = re.sub(r"[^A-Za-z0-9._-]+", "_", Path(uploaded_capture.name).stem)[:80]
stored_name = f"{stem or 'capture'}-{uuid4().hex[:12]}{suffix}"
storage = FileSystemStorage(location=settings.CAPTURE_UPLOAD_DIR)
stored_name = storage.save(stored_name, uploaded_capture)
try:
capture = create_capture(filename=stored_name, **values)
except ObjectDoesNotExist:
storage.delete(stored_name)
return Response({"error": "The selected device does not exist."}, status=400)
except ValidationError as exc:
storage.delete(stored_name)
return _validation_error(exc)
except Exception:
storage.delete(stored_name)
raise
return Response(
{"status": "success", "data": capture.to_json()},
status=status.HTTP_201_CREATED,
)
class ManufacturersView(APIView):
def get(self, request):
return Response(get_all_manufacturers())
class DeviceTypesView(APIView):
def get(self, request):
return Response(get_all_device_types())
class DevicesView(APIView):
def get(self, request):
return Response(get_all_devices())
class DeviceDetailView(APIView):
def get(self, request, device_id):
get_object_or_404(Device, id=device_id)
return Response(get_device_by_id(device_id))
class DeviceCapturesView(APIView):
def get(self, request, device_id):
get_object_or_404(Device, id=device_id)
return Response(get_files_by_device(device_id))
class DeleteDeviceView(APIView):
permission_classes = [IsAdminUser]
def delete(self, request, device_id):
device = get_object_or_404(Device, id=device_id)
device.delete()
return Response({"status": "success"})
class DeleteManufacturerView(APIView):
permission_classes = [IsAdminUser]
def delete(self, request, manufacturer_id):
item = get_object_or_404(Manufacturer, id=manufacturer_id)
try:
item.delete()
except ProtectedError:
return Response(
{"error": "The manufacturer is still used by a product."},
status=status.HTTP_409_CONFLICT,
)
return Response({"status": "success"})
class DeleteDeviceTypeView(APIView):
permission_classes = [IsAdminUser]
def delete(self, request, device_type_id):
item = get_object_or_404(DeviceType, id=device_type_id)
try:
item.delete()
except ProtectedError:
return Response(
{"error": "The device type is still used by a product."},
status=status.HTTP_409_CONFLICT,
)
return Response({"status": "success"})
class FindingsView(APIView):
def get(self, request, file_id):
get_object_or_404(File, id=file_id)
report = FindingReport.objects.filter(file_id=file_id).first()
if report is None:
return Response(
{
"status": "empty",
"message": "No security analysis is available for this capture.",
"summary": {
"total": 0,
"by_severity": {},
"by_category": {},
"highlights": [],
"top": [],
},
"findings": [],
}
)
return Response(json.loads(report.result_json))
class CloudResultsView(APIView):
def get(self, request, file_id):
get_object_or_404(File, id=file_id)
response = Response(collect_cloud_results(file_id))
response["Cache-Control"] = "no-store"
return response
class StartAnalysisView(APIView):
def post(self, request, file_id):
get_object_or_404(File, id=file_id)
try:
url_ids = [int(value) for value in (request.data.get("url_ids") or [])]
hash_ids = [int(value) for value in (request.data.get("hash_ids") or [])]
except (TypeError, ValueError):
return Response({"error": "Artifact IDs must be integers."}, status=400)
if not url_ids and not hash_ids:
return Response({"error": "Select at least one artifact."}, status=400)
from .analyze.deep_analysis import start_deep_analysis
start_deep_analysis(file_id, url_ids=url_ids, hash_ids=hash_ids)
return Response(
{"status": "started", "url_count": len(url_ids), "hash_count": len(hash_ids)},
status=status.HTTP_202_ACCEPTED,
)
class AnalysisStatusView(APIView):
def get(self, request, file_id):
get_object_or_404(File, id=file_id)
job = DeepAnalysisJob.objects.filter(file_id=file_id).first()
data = job.to_json() if job else {
"state": "pending",
"url_total": 0,
"url_done": 0,
"hash_total": 0,
"hash_done": 0,
"cracked": 0,
"progress_done": 0,
"progress_total": 0,
"percent": 0,
"message": "Not started.",
}
response = Response(data)
response["Cache-Control"] = "no-store"
return response
class ReportView(APIView):
def get(self, request, file_id, fmt):
from analyzer import FORMATS, render, result_from_dict
fmt = fmt.lower()
if fmt not in FORMATS:
return Response({"error": f"Allowed formats: {', '.join(FORMATS)}"}, status=400)
report = get_object_or_404(FindingReport, file_id=file_id)
result = result_from_dict(json.loads(report.result_json))
content = render(result, fmt, collect_cloud_results(file_id))
content_types = {"md": "text/markdown", "txt": "text/plain", "json": "application/json"}
response = HttpResponse(content, content_type=content_types[fmt])
response["Content-Disposition"] = (
f'attachment; filename="iot-pretester-4.6-demo-report-{file_id}.{fmt}"'
)
return response
+9
View File
@@ -0,0 +1,9 @@
"""ASGI entry point for the demo backend."""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "IoTPreTester.settings")
application = get_asgi_application()
+131
View File
@@ -0,0 +1,131 @@
"""Django settings for IoT PreTester 4.6 Demo."""
import os
from pathlib import Path
from django.core.exceptions import ImproperlyConfigured
BASE_DIR = Path(__file__).resolve().parent.parent
def env_bool(name: str, default: bool = False) -> bool:
return os.environ.get(name, str(default)).strip().lower() in {"1", "true", "yes"}
DEBUG = env_bool("DEBUG", False)
SECRET_KEY = os.environ.get("SECRET_KEY")
if not SECRET_KEY:
if DEBUG:
SECRET_KEY = "django-insecure-iot-pretester-4.6-demo-local-only"
else:
raise ImproperlyConfigured("SECRET_KEY must be configured when DEBUG is disabled.")
ALLOWED_HOSTS = [
host.strip()
for host in os.environ.get("ALLOWED_HOSTS", "localhost,127.0.0.1").split(",")
if host.strip()
]
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"corsheaders",
"rest_framework",
"IoTAnalyzer",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "IoTPreTester.urls"
WSGI_APPLICATION = "IoTPreTester.wsgi.application"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
]
},
}
]
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": os.environ.get("DATABASE_PATH", BASE_DIR / "db.sqlite3"),
"OPTIONS": {"timeout": 30},
}
}
AUTH_PASSWORD_VALIDATORS = [
{"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
]
PASSWORD_HASHERS = [
"django.contrib.auth.hashers.BCryptSHA256PasswordHasher",
"django.contrib.auth.hashers.PBKDF2PasswordHasher",
]
REST_FRAMEWORK = {
"DEFAULT_AUTHENTICATION_CLASSES": [
"IoTAnalyzer.authentication.DemoSessionAuthentication",
],
"DEFAULT_PERMISSION_CLASSES": [
"rest_framework.permissions.IsAuthenticated",
],
"DEFAULT_THROTTLE_CLASSES": [
"rest_framework.throttling.UserRateThrottle",
],
"DEFAULT_THROTTLE_RATES": {"user": "20/second"},
}
CORS_ALLOW_ALL_ORIGINS = env_bool("CORS_ALLOW_ALL_ORIGINS", False)
CORS_ALLOWED_ORIGINS = [
origin.strip()
for origin in os.environ.get("CORS_ALLOWED_ORIGINS", "http://localhost:3000").split(",")
if origin.strip()
]
CORS_ALLOW_CREDENTIALS = True
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
STATIC_URL = "static/"
STATIC_ROOT = BASE_DIR / "staticfiles"
CAPTURE_UPLOAD_DIR = Path(
os.environ.get("CAPTURE_UPLOAD_DIR", BASE_DIR.parent / "uploads")
)
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
APPEND_SLASH = False
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SAMESITE = "Lax"
X_FRAME_OPTIONS = "DENY"
SECURE_CONTENT_TYPE_NOSNIFF = True
if not DEBUG:
SESSION_COOKIE_SECURE = env_bool("SESSION_COOKIE_SECURE", False)
CSRF_COOKIE_SECURE = env_bool("CSRF_COOKIE_SECURE", False)
+9
View File
@@ -0,0 +1,9 @@
"""Root routes for the demo backend."""
from django.contrib import admin
from django.urls import include, path
urlpatterns = [
path("admin/", admin.site.urls),
path("", include("IoTAnalyzer.urls")),
]
+9
View File
@@ -0,0 +1,9 @@
"""WSGI entry point used by Gunicorn."""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "IoTPreTester.settings")
application = get_wsgi_application()
View File
+55
View File
@@ -0,0 +1,55 @@
# IoT PreTester 4.6 Demo analysis core
`analyzer` is the framework-independent network-capture analysis package used by both the Django backend and the command-line interface. It accepts `.pcap` or `.pcapng` files, normalizes protocol data once, runs independent detectors, and renders Markdown, text, or JSON reports.
## Main modules
- `context.py` defines the normalized `CaptureContext` records.
- `extract_pyshark.py` builds the context through tshark/pyshark.
- `detectors/` contains one detection concern per module.
- `engine.py` runs detectors and assembles the final result.
- `scoring.py` deduplicates and prioritizes findings.
- `provenance.py` attaches artifacts and packet/stream references.
- `artifacts.py` extracts URL and hash artifacts from the shared context.
- `enrich/` provides optional URL probing and hash recovery.
- `reporting/` renders `md`, `txt`, and `json` output.
## Command line
From `django_project/`:
```bash
python -m analyzer.cli ../demo_files/demo_capture.pcapng
python -m analyzer.cli capture.pcapng -f json -o report.json
python -m analyzer.cli capture.pcapng -f txt --packet-limit 100000
```
Exit codes are `0` for no high/critical findings, `2` when at least one high/critical finding exists, and `1` for execution errors.
## Python API
```python
from analyzer import AnalysisEngine, render
from analyzer.context import CaptureContext
result = AnalysisEngine().analyze_pcap("capture.pcapng")
print(render(result, "md"))
# Detector logic can also be tested without tshark.
synthetic_result = AnalysisEngine().analyze_context(CaptureContext())
```
## Tests
```bash
python -m unittest analyzer.tests.test_analyzer -v
```
The unit suite uses synthetic contexts and does not require tshark.
## Adding a detector
1. Add a focused module under `detectors/`.
2. Inherit from `BaseDetector`, apply `@register`, and return shared `Finding` objects from `detect()`.
3. Import the module from `detectors/__init__.py`.
4. Add focused tests under `tests/`.
+37
View File
@@ -0,0 +1,37 @@
"""Framework-independent analysis core for IoT PreTester 4.6 Demo.
The package is independent of Django and supports both backend and CLI use.
The pyshark/tshark extractor is loaded only when a capture is analyzed.
Headless example:
from analyzer import AnalysisEngine, render
result = AnalysisEngine().analyze_pcap("capture.pcapng")
print(render(result, "md"))
Synthetic-context example without tshark:
from analyzer import AnalysisEngine
from analyzer.context import CaptureContext
result = AnalysisEngine().analyze_context(CaptureContext(...))
"""
from __future__ import annotations
from .findings import Finding, Category, Severity, Confidence
from .context import CaptureContext
from .engine import AnalysisEngine, AnalysisResult, result_from_dict
from .reporting import render, FORMATS
from .artifacts import (
UrlArtifact, HashArtifact, extract_urls, extract_hashes, identify_hash_mode,
)
# Importing the package registers every detector.
from . import detectors # noqa: F401
__version__ = "4.6.0-demo.0"
__all__ = [
"Finding", "Category", "Severity", "Confidence",
"CaptureContext", "AnalysisEngine", "AnalysisResult", "result_from_dict",
"render", "FORMATS", "__version__",
"UrlArtifact", "HashArtifact", "extract_urls", "extract_hashes", "identify_hash_mode",
]
+237
View File
@@ -0,0 +1,237 @@
"""Extract endpoint and crackable hash artifacts from a capture context.
Extraction is independent of Django, pyshark, and I/O. Active endpoint probing
and hash cracking remain isolated in ``analyzer.enrich``.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from typing import Optional
from .context import CaptureContext
# --------------------------------------------------------------------------- #
# Artifact data models
# --------------------------------------------------------------------------- #
@dataclass
class UrlArtifact:
"""A domain, host, or URL observed in the capture."""
value: str
kind: str = "unknown" # "dns" | "sni" | "http_host" | "payload"
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
timestamp: Optional[float] = None
packet_number: Optional[int] = None
# Passive observations are preferred over active probing results.
observed_ports: set[int] = field(default_factory=set)
observed_tls: set[str] = field(default_factory=set)
def to_dict(self) -> dict:
return {
"value": self.value, "kind": self.kind,
"src_ip": self.src_ip, "dst_ip": self.dst_ip,
"timestamp": self.timestamp, "packet_number": self.packet_number,
"observed_ports": sorted(self.observed_ports),
"observed_tls": sorted(self.observed_tls),
}
@dataclass
class HashArtifact:
"""A hash or crackable authentication artifact from the capture."""
value: str # Raw hash or complete hashcat line
hashcat_mode: Optional[int] = None
hash_format: Optional[str] = None # Human-readable format name
kind: str = "raw_hex" # "raw_hex" | "netntlmv2" | "krb5asrep" | "krb5tgs" | "wpa" | "digest"
account: Optional[str] = None
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
timestamp: Optional[float] = None
packet_number: Optional[int] = None
def to_dict(self) -> dict:
return {
"value": self.value, "hashcat_mode": self.hashcat_mode,
"hash_format": self.hash_format, "kind": self.kind,
"account": self.account, "src_ip": self.src_ip, "dst_ip": self.dst_ip,
"timestamp": self.timestamp, "packet_number": self.packet_number,
}
# --------------------------------------------------------------------------- #
# Hashcat mode detection
# --------------------------------------------------------------------------- #
# Hex length to hashcat mode and format name.
_LEN_MODES = {
32: (0, "md5"),
40: (100, "sha1"),
56: (300, "sha224"),
64: (1400, "sha256"),
96: (6100, "sha384"),
128: (1700, "sha512"),
}
# Structured format name to hashcat mode.
_FORMAT_MODES = {
"netntlmv1": 5500,
"netntlmv2": 5600,
"krb5asrep": 18200,
"krb5tgs": 13100,
"wpa": 22000,
"eapol": 22000,
}
_NETNTLMV2_RE = re.compile(r"^[^:]+::[^:]*:[0-9a-fA-F]{16}:[0-9a-fA-F]{32}:[0-9a-fA-F]+$")
_NETNTLMV1_RE = re.compile(r"^[^:]+::[^:]*:[0-9a-fA-F]{48}:[0-9a-fA-F]{48}:[0-9a-fA-F]{16}$")
def identify_hash_mode(value: str, hint_format: Optional[str] = None):
"""Return a hashcat mode and format using structure before hex length."""
v = (value or "").strip()
if not v:
return None, None
if hint_format:
base = str(hint_format).split("(")[0].strip().lower()
if base in _FORMAT_MODES:
return _FORMAT_MODES[base], hint_format
low = v.lower()
if low.startswith("$krb5asrep$"):
return 18200, "krb5asrep"
if low.startswith("$krb5tgs$"):
return 13100, "krb5tgs"
if low.startswith("$wpapsk$") or low.startswith("wpa*") or low.startswith("$wpa$"):
return 22000, "wpa"
if _NETNTLMV2_RE.match(v):
return 5600, "NetNTLMv2"
if _NETNTLMV1_RE.match(v):
return 5500, "NetNTLMv1"
if all(c in "0123456789abcdefABCDEF" for c in v) and len(v) in _LEN_MODES:
return _LEN_MODES[len(v)]
return None, None
def _kind_from_mode(mode: Optional[int]) -> str:
return {
5500: "netntlmv1", 5600: "netntlmv2",
18200: "krb5asrep", 13100: "krb5tgs", 22000: "wpa",
}.get(mode, "raw_hex")
# --------------------------------------------------------------------------- #
# Endpoint extraction
# --------------------------------------------------------------------------- #
_URL_RE = re.compile(
r"\b(?:"
r"(?:https?://|www\.)(?:[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6})" # URL or www host
r"|"
r"(?:[a-zA-Z0-9-]+\.){2,}[a-zA-Z]{2,6}" # Bare multi-label domain
r"|"
r"https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" # URL with IPv4 host
r")"
r"(?:/[a-zA-Z0-9._%/-]*)?\b"
)
def _normalize_host(value: Optional[str]) -> Optional[str]:
if not value:
return None
v = str(value).strip().lower()
v = v.rstrip(".")
return v or None
def extract_urls(ctx: CaptureContext) -> list[UrlArtifact]:
"""Collect endpoints and attach passively observed ports and TLS versions."""
by_value: dict[str, UrlArtifact] = {}
def add(value, kind, **kw) -> Optional[UrlArtifact]:
v = _normalize_host(value)
if not v:
return None
art = by_value.get(v)
if art is None:
art = UrlArtifact(value=v, kind=kind, **kw)
by_value[v] = art
return art
for q in ctx.dns_queries:
add(q.query_name, "dns", src_ip=q.src_ip, dst_ip=q.dst_ip,
timestamp=q.timestamp, packet_number=q.packet_number)
for t in ctx.tls_sessions:
if t.sni:
art = add(t.sni, "sni", src_ip=t.src_ip, dst_ip=t.dst_ip,
timestamp=t.timestamp, packet_number=t.packet_number)
if art:
if t.dst_port:
art.observed_ports.add(t.dst_port)
if t.version:
art.observed_tls.add(t.version)
for r in ctx.http_requests:
if r.host:
art = add(r.host, "http_host", src_ip=r.src_ip, dst_ip=r.dst_ip,
timestamp=r.timestamp, packet_number=r.packet_number)
if art and r.dst_port:
art.observed_ports.add(r.dst_port)
for p in ctx.text_payloads:
text = p.get("text") or ""
for m in _URL_RE.findall(text):
add(m, "payload", src_ip=p.get("src_ip"), dst_ip=p.get("dst_ip"),
timestamp=p.get("timestamp"), packet_number=p.get("packet_number"))
return list(by_value.values())
# --------------------------------------------------------------------------- #
# Hash extraction
# --------------------------------------------------------------------------- #
_HASH_HEX_RE = re.compile(r"\b[a-fA-F0-9]{32,64}\b")
def extract_hashes(ctx: CaptureContext) -> list[HashArtifact]:
"""Collect structured authentication material and raw hexadecimal hashes."""
out: list[HashArtifact] = []
seen: set[str] = set()
def add(value, *, mode, fmt, kind, **kw) -> None:
if not value or value in seen:
return
seen.add(value)
out.append(HashArtifact(value=value, hashcat_mode=mode, hash_format=fmt,
kind=kind, **kw))
# Structured material already reconstructed by the packet extractor.
for am in ctx.auth_material:
if getattr(am, "hashcat", None):
mode = am.hashcat_mode
fmt = am.hash_format
if mode is None:
mode, fmt = identify_hash_mode(am.hashcat, hint_format=fmt)
add(am.hashcat, mode=mode, fmt=fmt, kind=_kind_from_mode(mode),
account=am.account, src_ip=am.src_ip, dst_ip=am.dst_ip,
timestamp=am.timestamp, packet_number=am.packet_number)
# Raw 32–64 character hex values use a mode only when the length is known.
for p in ctx.text_payloads:
text = p.get("text") or ""
for m in _HASH_HEX_RE.findall(text):
mode, fmt = identify_hash_mode(m)
add(m.lower(), mode=mode, fmt=fmt, kind="raw_hex",
src_ip=p.get("src_ip"), dst_ip=p.get("dst_ip"),
timestamp=p.get("timestamp"), packet_number=p.get("packet_number"))
return out
+72
View File
@@ -0,0 +1,72 @@
"""Headless CLI for IoT PreTester 4.6 Demo.
Examples:
python -m analyzer.cli capture.pcapng
python -m analyzer.cli capture.pcapng --format json -o report.json
python -m analyzer.cli capture.pcapng --format txt --packet-limit 50000
Exit codes:
0 Analysis completed without high or critical findings
2 Analysis completed with at least one high or critical finding
1 Capture or runtime dependency error
"""
from __future__ import annotations
import argparse
import sys
from . import __version__
from .engine import AnalysisEngine
from .findings import Severity
from .reporting import render, FORMATS
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(
prog="analyzer",
description="Analyze the security of a PCAP or PCAPNG network capture.",
)
p.add_argument("capture", help="Path to a .pcap or .pcapng file")
p.add_argument("-f", "--format", choices=FORMATS, default="md",
help="Output format (default: md)")
p.add_argument("-o", "--output", default=None,
help="Output file (default: standard output)")
p.add_argument("--packet-limit", type=int, default=None,
help="Maximum packets to read from a large capture")
p.add_argument(
"--version",
action="version",
version=f"IoT PreTester 4.6 Demo analyzer {__version__}",
)
return p
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
try:
result = AnalysisEngine().analyze_pcap(args.capture, packet_limit=args.packet_limit)
except FileNotFoundError as exc:
print(f"Error: {exc}", file=sys.stderr)
return 1
except RuntimeError as exc:
print(f"Error: {exc}", file=sys.stderr)
return 1
report = render(result, args.format)
if args.output:
with open(args.output, "w", encoding="utf-8") as fh:
fh.write(report)
print(f"Report written: {args.output} "
f"({result.meta.get('finding_count', 0)} Findings)", file=sys.stderr)
else:
print(report)
worst = max((f.severity.rank for f in result.findings), default=0)
return 2 if worst >= Severity.HIGH.rank else 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())
+249
View File
@@ -0,0 +1,249 @@
"""Normalized capture data consumed by every detector.
The module is independent of Django and pyshark so detector logic can be tested
with synthetic data. ``extract_pyshark.py`` performs the actual PCAP extraction.
"""
from __future__ import annotations
import ipaddress
from dataclasses import dataclass, field
from typing import Optional
# --------------------------------------------------------------------------- #
# Normalized records
# --------------------------------------------------------------------------- #
@dataclass
class Host:
ip: str
macs: set[str] = field(default_factory=set)
hostnames: set[str] = field(default_factory=set)
bytes_sent: int = 0
bytes_received: int = 0
ports_offered: set[int] = field(default_factory=set) # Ports on which the host responds
ports_contacted: set[int] = field(default_factory=set) # Ports contacted by the host
@property
def is_private(self) -> bool:
return _is_private(self.ip)
@dataclass
class Conversation:
"""A five-tuple flow aggregated between two endpoints."""
src_ip: str
dst_ip: str
src_port: Optional[int]
dst_port: Optional[int]
protocol: str # For example, "tcp" or "udp"
packets: int = 0
bytes: int = 0
bytes_src_to_dst: int = 0
bytes_dst_to_src: int = 0
first_seen: Optional[float] = None
last_seen: Optional[float] = None
stream_id: Optional[int] = None
timestamps: list[float] = field(default_factory=list) # Used to detect beaconing
packet_numbers: list[int] = field(default_factory=list) # Bounded evidence packet list
@dataclass
class DnsQuery:
query_name: str
qtype: Optional[str] = None
response_ips: list[str] = field(default_factory=list)
rcode: Optional[str] = None # For example, "NOERROR" or "NXDOMAIN"
timestamp: Optional[float] = None
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
resolver: Optional[str] = None
packet_number: Optional[int] = None
@dataclass
class HttpRequest:
method: Optional[str] = None
host: Optional[str] = None
uri: Optional[str] = None
user_agent: Optional[str] = None
authorization: Optional[str] = None # Authorization header value
cookies: Optional[str] = None
headers: dict[str, str] = field(default_factory=dict)
body: Optional[str] = None
request_bytes: int = 0
is_tls: bool = False # True when only TLS metadata is visible
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
dst_port: Optional[int] = None
timestamp: Optional[float] = None
stream_id: Optional[int] = None
packet_number: Optional[int] = None
@dataclass
class HttpResponse:
status_code: Optional[int] = None
content_type: Optional[str] = None
server: Optional[str] = None
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
timestamp: Optional[float] = None
stream_id: Optional[int] = None
packet_number: Optional[int] = None
@dataclass
class TlsSession:
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
dst_port: Optional[int] = 443
version: Optional[str] = None # For example, "TLS 1.2" or "TLS 1.0"
cipher: Optional[str] = None # Cipher-suite name
sni: Optional[str] = None
cert_subject_cn: Optional[str] = None
cert_issuer_cn: Optional[str] = None
cert_not_after: Optional[str] = None
cert_self_signed: Optional[bool] = None
cert_expired: Optional[bool] = None
timestamp: Optional[float] = None
stream_id: Optional[int] = None
packet_number: Optional[int] = None
@dataclass
class AuthMaterial:
"""Hashes, challenge-response material, and authentication tickets."""
kind: str # "ntlmssp", "netntlmv2", "kerberos_asrep", ...
value: Optional[str] = None
account: Optional[str] = None
domain: Optional[str] = None
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
protocol: Optional[str] = None
timestamp: Optional[float] = None
packet_number: Optional[int] = None
# Crackable material, when reconstruction is possible.
hashcat: Optional[str] = None # Complete hashcat input line
hashcat_mode: Optional[int] = None # For example, 5600 (NetNTLMv2)
hash_format: Optional[str] = None # Human-readable format name
server_challenge: Optional[str] = None # NTLMSSP server challenge (hex)
nt_response: Optional[str] = None # NTLMv2 response (hex)
etype: Optional[str] = None # Kerberos encryption type
@dataclass
class ArpEvent:
sender_ip: Optional[str] = None
sender_mac: Optional[str] = None
target_ip: Optional[str] = None
opcode: Optional[int] = None # 1=request, 2=reply
is_gratuitous: bool = False
timestamp: Optional[float] = None
packet_number: Optional[int] = None
@dataclass
class FileTransfer:
filename: Optional[str] = None
extension: Optional[str] = None
content_type: Optional[str] = None
size: Optional[int] = None
protocol: Optional[str] = None # "http", "ftp-data", "smb", "tftp"
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
direction: Optional[str] = None # "download" / "upload"
timestamp: Optional[float] = None
stream_id: Optional[int] = None
packet_number: Optional[int] = None
@dataclass
class PlaintextAuth:
"""Clear-text authentication observed in an application protocol."""
protocol: str
username: Optional[str] = None
password: Optional[str] = None
src_ip: Optional[str] = None
dst_ip: Optional[str] = None
dst_port: Optional[int] = None
timestamp: Optional[float] = None
stream_id: Optional[int] = None
packet_number: Optional[int] = None
# --------------------------------------------------------------------------- #
# Central capture context
# --------------------------------------------------------------------------- #
@dataclass
class CaptureContext:
"""All detector input extracted during a single capture pass."""
path: Optional[str] = None
packet_count: int = 0
start_time: Optional[float] = None
end_time: Optional[float] = None
hosts: dict[str, Host] = field(default_factory=dict)
conversations: list[Conversation] = field(default_factory=list)
dns_queries: list[DnsQuery] = field(default_factory=list)
http_requests: list[HttpRequest] = field(default_factory=list)
http_responses: list[HttpResponse] = field(default_factory=list)
tls_sessions: list[TlsSession] = field(default_factory=list)
auth_material: list[AuthMaterial] = field(default_factory=list)
arp_events: list[ArpEvent] = field(default_factory=list)
file_transfers: list[FileTransfer] = field(default_factory=list)
plaintext_auth: list[PlaintextAuth] = field(default_factory=list)
# protocol_stats: protocol-name -> packet count
protocol_stats: dict[str, int] = field(default_factory=dict)
# Generic text payloads with packet metadata for secret detection.
text_payloads: list[dict] = field(default_factory=list)
@property
def duration(self) -> float:
if self.start_time is None or self.end_time is None:
return 0.0
return max(0.0, self.end_time - self.start_time)
def host(self, ip: str) -> Host:
h = self.hosts.get(ip)
if h is None:
h = Host(ip=ip)
self.hosts[ip] = h
return h
def is_external(self, ip: Optional[str]) -> bool:
"""Return whether an address is globally routable."""
if not ip:
return False
return not _is_private(ip)
def protocol_count(self, *names: str) -> int:
"""Return a case-insensitive packet total for the requested protocols."""
wanted = {n.lower() for n in names}
return sum(
count for proto, count in self.protocol_stats.items()
if str(proto).lower() in wanted
)
def _is_private(ip: str) -> bool:
try:
addr = ipaddress.ip_address(ip)
except ValueError:
return False
return (
addr.is_private
or addr.is_loopback
or addr.is_link_local
or addr.is_multicast
or addr.is_reserved
or addr.is_unspecified
)
+122
View File
@@ -0,0 +1,122 @@
"""Pure helpers that format captured authentication material for hashcat.
Supported hashcat modes:
5500 NetNTLMv1
5600 NetNTLMv2
18200 Kerberos AS-REP (etype 23)
13100 Kerberos TGS-REP (etype 23)
"""
from __future__ import annotations
import base64
from typing import Optional
def _hex(s: Optional[str]) -> str:
"""Normalize a hexadecimal string and remove separators."""
if not s:
return ""
return "".join(c for c in str(s).lower() if c in "0123456789abcdef")
# --------------------------------------------------------------------------- #
# NTLM / NetNTLM
# --------------------------------------------------------------------------- #
def build_netntlmv2(user, domain, server_challenge, nt_response) -> Optional[dict]:
"""NetNTLMv2 (hashcat -m 5600).
Format: user::domain:serverchallenge:NTproofstr:blob
The NT response contains a 16-byte proof followed by the remaining blob.
"""
chal = _hex(server_challenge)
nt = _hex(nt_response)
if not user or len(chal) != 16 or len(nt) < 32:
return None
ntproof, blob = nt[:32], nt[32:]
line = f"{user}::{domain or ''}:{chal}:{ntproof}:{blob}"
return {"hashcat": line, "hashcat_mode": 5600, "format": "NetNTLMv2"}
def build_netntlmv1(user, host, server_challenge, lm_response, nt_response) -> Optional[dict]:
"""NetNTLMv1 (hashcat -m 5500).
Format: user::host:LMresponse:NTresponse:serverchallenge
"""
chal = _hex(server_challenge)
nt = _hex(nt_response)
lm = _hex(lm_response) or nt
if not user or len(chal) != 16 or len(nt) < 32:
return None
line = f"{user}::{host or ''}:{lm}:{nt}:{chal}"
return {"hashcat": line, "hashcat_mode": 5500, "format": "NetNTLMv1"}
# --------------------------------------------------------------------------- #
# Kerberos
# --------------------------------------------------------------------------- #
def build_krb5_asrep(user, realm, etype, cipher) -> Optional[dict]:
"""Kerberos AS-REP Roasting (hashcat -m 18200, etype 23).
Format: $krb5asrep$<etype>$user@REALM:<checksum>$<edata>
cipher = checksum (16 Byte) + edata.
"""
ci = _hex(cipher)
if not user or len(ci) <= 32:
return None
et = str(etype or 23)
checksum, edata = ci[:32], ci[32:]
realm_u = (realm or "").upper()
line = f"$krb5asrep${et}${user}@{realm_u}:{checksum}${edata}"
mode = 18200 if et == "23" else None
return {"hashcat": line, "hashcat_mode": mode, "format": f"krb5asrep(etype={et})"}
def build_krb5_tgsrep(user, realm, spn, etype, cipher) -> Optional[dict]:
"""Kerberoasting / TGS-REP (hashcat -m 13100, etype 23).
Format: $krb5tgs$<etype>$*user$realm$spn*$<checksum>$<edata>
"""
ci = _hex(cipher)
if not user or len(ci) <= 32:
return None
et = str(etype or 23)
checksum, edata = ci[:32], ci[32:]
realm_u = (realm or "").upper()
line = f"$krb5tgs${et}$*{user}${realm_u}${spn or ''}*${checksum}${edata}"
mode = 13100 if et == "23" else None
return {"hashcat": line, "hashcat_mode": mode, "format": f"krb5tgs(etype={et})"}
# --------------------------------------------------------------------------- #
# SMTP AUTH is clear text with Base64 encoding only.
# --------------------------------------------------------------------------- #
def _b64(token: str) -> Optional[str]:
try:
return base64.b64decode(str(token).strip() + "===").decode("utf-8", "ignore")
except Exception:
return None
def decode_smtp_auth_plain(token: str) -> Optional[tuple[Optional[str], Optional[str]]]:
"""Decode an AUTH PLAIN token into username and password."""
raw = _b64(token)
if raw is None:
return None
parts = raw.split("\x00")
if len(parts) >= 3:
return parts[1] or None, parts[2] or None
if len(parts) == 2:
return parts[0] or None, parts[1] or None
return None
def decode_smtp_auth_login(user_token: Optional[str],
pass_token: Optional[str]) -> tuple[Optional[str], Optional[str]]:
"""Decode the separately encoded AUTH LOGIN username and password."""
user = _b64(user_token) if user_token else None
pw = _b64(pass_token) if pass_token else None
return user, pw
@@ -0,0 +1,17 @@
"""Import and register the ordered default detector set."""
from .base import BaseDetector, register, build_all, all_detector_classes # noqa: F401
# Import order defines the stable detector order in reports.
from . import plaintext_credentials # 1
from . import password_hashes # 2
from . import tokens_secrets # 3
from . import malware_beaconing # 4
from . import exfiltration # 5
from . import mitm # 6
from . import insecure_protocols # 7
from . import tls_weakness # 8
from . import dns_anomaly # 9
from . import bruteforce_scan_filetransfer # 10
__all__ = ["BaseDetector", "register", "build_all", "all_detector_classes"]
+45
View File
@@ -0,0 +1,45 @@
"""Detector base class and registration mechanism."""
from __future__ import annotations
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding
class BaseDetector:
#: Short unique name referenced by findings and reports.
name: str = "BaseDetector"
#: Human-readable description of the detector scope.
description: str = ""
def detect(self, ctx: CaptureContext) -> Iterable[Finding]: # pragma: no cover
raise NotImplementedError
def _tag(self, findings: Iterable[Finding]) -> list[Finding]:
out = []
for f in findings:
f.detector = self.name
out.append(f)
return out
# ---- Registry ------------------------------------------------------------- #
_REGISTRY: dict[str, type[BaseDetector]] = {}
def register(cls: type[BaseDetector]) -> type[BaseDetector]:
"""Register a detector class."""
_REGISTRY[cls.name] = cls
return cls
def all_detector_classes() -> list[type[BaseDetector]]:
return list(_REGISTRY.values())
def build_all() -> list[BaseDetector]:
"""Instantiate all registered detector classes."""
return [cls() for cls in _REGISTRY.values()]
@@ -0,0 +1,154 @@
"""Detect brute force, scanning, and suspicious file transfers."""
from __future__ import annotations
import os
from collections import defaultdict
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
# File extensions that warrant closer inspection.
_SUSPICIOUS_EXT = {
".exe", ".dll", ".ps1", ".vbs", ".js", ".jar", ".bat", ".cmd", ".scr",
".hta", ".lnk", ".docm", ".xlsm", ".pptm", ".zip", ".rar", ".7z", ".iso", ".elf",
}
_PORTSCAN_PORTS = 15
_HOSTSWEEP_HOSTS = 15
_FAILED_LOGINS = 5
@register
class BruteforceScanFileTransferDetector(BaseDetector):
name = "BruteforceScanFileTransferDetector"
description = "Detects scans, repeated authentication failures, and risky downloads."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
ports_per_pair: dict[tuple, set[int]] = defaultdict(set)
hosts_per_src: dict[str, set[str]] = defaultdict(set)
packets_per_pair: dict[tuple, list] = defaultdict(list)
packets_per_src: dict[str, list] = defaultdict(list)
for conv in ctx.conversations:
if conv.src_ip and conv.dst_ip:
if conv.dst_port:
ports_per_pair[(conv.src_ip, conv.dst_ip)].add(conv.dst_port)
hosts_per_src[conv.src_ip].add(conv.dst_ip)
if conv.packet_numbers:
packets_per_pair[(conv.src_ip, conv.dst_ip)].append(conv.packet_numbers[0])
packets_per_src[conv.src_ip].append(conv.packet_numbers[0])
for (src, dst), ports in ports_per_pair.items():
if len(ports) >= _PORTSCAN_PORTS:
findings.append(Finding(
title="Port scan detected",
category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
source_ip=src, destination_ip=dst,
risk_description=(
"One host contacted many ports on a target, which is characteristic "
"of network reconnaissance."
),
recommendation="Review the source and block unauthorized external scanners.",
evidence={"distinct_ports": len(ports), "dedup_key": f"portscan:{src}:{dst}"},
related_packets=sorted(set(packets_per_pair.get((src, dst), [])))[:8],
))
for src, dsts in hosts_per_src.items():
if len(dsts) >= _HOSTSWEEP_HOSTS:
findings.append(Finding(
title="Host sweep detected",
category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
severity=Severity.MEDIUM, confidence=Confidence.LOW,
source_ip=src,
risk_description=(
"One host contacted many different targets, which may indicate a "
"network sweep or worm behavior."
),
recommendation="Review the source and investigate internal hosts for compromise.",
evidence={"distinct_hosts": len(dsts), "dedup_key": f"sweep:{src}"},
related_packets=sorted(set(packets_per_src.get(src, [])))[:8],
))
# Repeated HTTP authorization failures are a brute-force indicator.
http_401: dict[tuple, int] = defaultdict(int)
http_401_pkts: dict[tuple, list] = defaultdict(list)
for resp in ctx.http_responses:
if resp.status_code in (401, 403):
http_401[(resp.dst_ip, resp.src_ip)] += 1
if resp.packet_number:
http_401_pkts[(resp.dst_ip, resp.src_ip)].append(resp.packet_number)
for (client, server), count in http_401.items():
if count >= _FAILED_LOGINS:
findings.append(Finding(
title="Repeated failed HTTP authentication",
category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
source_ip=client, destination_ip=server, protocol="http",
risk_description=(
"Many HTTP 401 or 403 responses may indicate a password guessing attack."
),
recommendation="Enable rate limiting or lockout controls and review the source.",
evidence={"failed_responses": count, "dedup_key": f"bruteforce:{client}:{server}"},
related_packets=sorted(set(http_401_pkts.get((client, server), [])))[:8],
))
seen_files: set[str] = set()
for req in ctx.http_requests:
uri = (req.uri or "").split("?")[0]
ext = os.path.splitext(uri)[1].lower()
if ext in _SUSPICIOUS_EXT:
key = f"file:{req.host}:{uri}"
if key in seen_files:
continue
seen_files.add(key)
findings.append(Finding(
title=f"Potentially harmful file download ({ext})",
category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
source_ip=req.src_ip, destination_ip=req.dst_ip, protocol="http", port=req.dst_port,
timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
risk_description=(
"An executable or script-capable file was transferred over HTTP and "
"may represent a malware download."
),
recommendation="Analyze the file in a sandbox and check its hash against threat intelligence.",
evidence={"host": req.host, "uri": uri, "extension": ext, "dedup_key": key},
related_packets=[p for p in [req.packet_number] if p],
))
# Response metadata can reveal risky files absent from the request URI.
for ft in ctx.file_transfers:
ext = (ft.extension or "").lower()
ct = (ft.content_type or "").lower()
is_executable_ct = any(
m in ct for m in ("x-msdownload", "x-dosexec", "x-executable",
"portable-executable", "x-elf", "x-msi")
)
if ext not in _SUSPICIOUS_EXT and not is_executable_ct:
continue
key = f"ft:{ft.dst_ip}:{ft.filename or ct}"
if key in seen_files:
continue
seen_files.add(key)
label = ext or ct or "binary file"
findings.append(Finding(
title=f"Potentially harmful file download ({label})",
category=Category.BRUTEFORCE_SCAN_FILETRANSFER,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
source_ip=ft.dst_ip, destination_ip=ft.src_ip, protocol=ft.protocol,
timestamp_first_seen=ft.timestamp, timestamp_last_seen=ft.timestamp,
risk_description=(
"An executable or script-capable file was transferred and may represent "
"a malware download."
),
recommendation="Analyze the file in a sandbox and check its hash against threat intelligence.",
evidence={"filename": ft.filename, "content_type": ft.content_type,
"extension": ext or None, "dedup_key": key},
related_packets=[p for p in [ft.packet_number] if p],
))
return self._tag(findings)
@@ -0,0 +1,154 @@
"""Detect anomalous DNS activity and potential DNS tunneling."""
from __future__ import annotations
import math
from collections import defaultdict
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
_LONG_LABEL = 40
_ENTROPY_MIN_LEN = 12
_HIGH_ENTROPY = 3.2
_NXDOMAIN_RATIO = 0.3
_MANY_SUBDOMAINS = 30
_TUNNEL_QTYPES = {"TXT", "NULL"}
_TXT_TUNNEL_MIN = 20
_TXT_TUNNEL_RATIO = 0.5
@register
class DnsAnomalyDetector(BaseDetector):
name = "DnsAnomalyDetector"
description = "Detects DNS tunneling, DGA-like names, and high NXDOMAIN rates."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
if not ctx.dns_queries:
return findings
nx = 0
total = 0
txt_like = 0
subdomains_per_domain: dict[str, set[str]] = defaultdict(set)
packets_per_domain: dict[str, list] = defaultdict(list)
nx_packets: list = []
txt_packets: list = []
flagged_random: set[str] = set()
for q in ctx.dns_queries:
name = (q.query_name or "").rstrip(".").lower()
if not name:
continue
total += 1
if (q.rcode or "").upper() == "NXDOMAIN":
nx += 1
if q.packet_number:
nx_packets.append(q.packet_number)
if (q.qtype or "").upper() in _TUNNEL_QTYPES:
txt_like += 1
if q.packet_number:
txt_packets.append(q.packet_number)
labels = name.split(".")
reg_domain = ".".join(labels[-2:]) if len(labels) >= 2 else name
if len(labels) > 2:
subdomains_per_domain[reg_domain].add(".".join(labels[:-2]))
if q.packet_number:
packets_per_domain[reg_domain].append(q.packet_number)
# Long or high-entropy labels may indicate a DGA or tunneling channel.
for label in labels[:-2] or labels:
if len(label) >= _LONG_LABEL or (
len(label) >= _ENTROPY_MIN_LEN and _entropy(label) >= _HIGH_ENTROPY
):
if reg_domain not in flagged_random:
flagged_random.add(reg_domain)
findings.append(Finding(
title="DNS query suspicious for DGA or tunneling activity",
category=Category.DNS_ANOMALY,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
source_ip=q.src_ip, destination_ip=q.dst_ip, protocol="dns", port=53,
timestamp_first_seen=q.timestamp, timestamp_last_seen=q.timestamp,
risk_description=(
"Long or random-looking subdomains are characteristic of DNS "
"tunneling and malware domain-generation algorithms."
),
recommendation=(
"Check the domain against threat intelligence and enable DNS "
"tunneling detection and egress controls."
),
evidence={"query_name": name, "registered_domain": reg_domain,
"label_entropy": round(_entropy(label), 2),
"dedup_key": f"dga:{reg_domain}"},
related_packets=[p for p in [q.packet_number] if p],
))
break
for domain, subs in subdomains_per_domain.items():
if len(subs) >= _MANY_SUBDOMAINS:
findings.append(Finding(
title=f"Unusually many subdomains under {domain}",
category=Category.DNS_ANOMALY,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
protocol="dns", port=53,
risk_description=(
"A large number of unique subdomains under one domain is a strong "
"indicator of DNS tunneling or exfiltration."
),
recommendation="Review or block the domain and enable tunneling detection.",
evidence={"registered_domain": domain, "unique_subdomains": len(subs),
"dedup_key": f"subtunnel:{domain}"},
related_packets=sorted(set(packets_per_domain.get(domain, [])))[:8],
))
# TXT and NULL records can carry the large payloads used by DNS tunnels.
if txt_like >= _TXT_TUNNEL_MIN and total and (txt_like / total) >= _TXT_TUNNEL_RATIO:
findings.append(Finding(
title="Unusually many TXT or NULL DNS queries",
category=Category.DNS_ANOMALY,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
protocol="dns", port=53,
risk_description=(
"TXT and NULL records can carry large payloads and are commonly abused "
"for DNS tunneling or data exfiltration."
),
recommendation=(
"Review the requesting source and destination domain, then enforce DNS "
"tunneling detection and egress controls."
),
evidence={"txt_null_queries": txt_like, "total_queries": total,
"ratio": round(txt_like / total, 2), "dedup_key": "txt_tunnel"},
related_packets=sorted(set(txt_packets))[:8],
))
if total >= 20 and (nx / total) >= _NXDOMAIN_RATIO:
findings.append(Finding(
title="High NXDOMAIN rate",
category=Category.DNS_ANOMALY,
severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
protocol="dns", port=53,
risk_description=(
"Repeated NXDOMAIN responses may indicate malware testing generated "
"command-and-control domains."
),
recommendation="Review the requesting source and investigate it for malware.",
evidence={"nxdomain": nx, "total_queries": total,
"ratio": round(nx / total, 2), "dedup_key": "nxdomain_rate"},
related_packets=sorted(set(nx_packets))[:8],
))
return self._tag(findings)
def _entropy(s: str) -> float:
if not s:
return 0.0
counts = defaultdict(int)
for ch in s:
counts[ch] += 1
n = len(s)
return -sum((c / n) * math.log2(c / n) for c in counts.values())
@@ -0,0 +1,125 @@
"""Detect potential data exfiltration."""
from __future__ import annotations
from collections import defaultdict
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
# Conservative heuristic thresholds.
_LARGE_UPLOAD_BYTES = 5 * 1024 * 1024
_AGG_UPLOAD_BYTES = 20 * 1024 * 1024
_MANY_SMALL_TRANSFERS = 50
_DNS_TXT_VOLUME = 100
@register
class ExfiltrationDetector(BaseDetector):
name = "ExfiltrationDetector"
description = "Detects unusual outbound volumes and transfer patterns."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
per_dest_bytes: dict[str, int] = defaultdict(int)
per_dest_count: dict[str, int] = defaultdict(int)
first_seen: dict[str, float] = {}
last_seen: dict[str, float] = {}
src_for_dest: dict[str, str] = {}
packets_for_dest: dict[str, list] = defaultdict(list)
streams_for_dest: dict[str, list] = defaultdict(list)
for conv in ctx.conversations:
# Evaluate outbound traffic from a private source to an external target.
if not (conv.src_ip and ctx.is_external(conv.dst_ip)
and not ctx.is_external(conv.src_ip)):
continue
up = conv.bytes_src_to_dst or conv.bytes
per_dest_bytes[conv.dst_ip] += up
per_dest_count[conv.dst_ip] += 1
src_for_dest[conv.dst_ip] = conv.src_ip
if conv.packet_numbers:
packets_for_dest[conv.dst_ip].append(conv.packet_numbers[0])
if conv.stream_id is not None:
streams_for_dest[conv.dst_ip].append(conv.stream_id)
if conv.first_seen is not None:
first_seen[conv.dst_ip] = min(first_seen.get(conv.dst_ip, conv.first_seen), conv.first_seen)
if conv.last_seen is not None:
last_seen[conv.dst_ip] = max(last_seen.get(conv.dst_ip, conv.last_seen), conv.last_seen)
if up >= _LARGE_UPLOAD_BYTES:
findings.append(Finding(
title="Large outbound transfer to an external target",
category=Category.EXFILTRATION,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
source_ip=conv.src_ip, destination_ip=conv.dst_ip,
protocol=conv.protocol, port=conv.dst_port,
timestamp_first_seen=conv.first_seen, timestamp_last_seen=conv.last_seen,
risk_description=(
"A large amount of data left the device for an external target, "
"which may indicate data loss."
),
recommendation="Validate the destination, content, and authorization.",
evidence={"uploaded_bytes": up,
"dedup_key": f"bigup:{conv.src_ip}:{conv.dst_ip}:{conv.dst_port}"},
related_packets=list(conv.packet_numbers),
related_streams=[s for s in [conv.stream_id] if s is not None],
))
for dest, total in per_dest_bytes.items():
if total >= _AGG_UPLOAD_BYTES:
findings.append(Finding(
title="High aggregate upload volume to an external target",
category=Category.EXFILTRATION,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
source_ip=src_for_dest.get(dest), destination_ip=dest,
timestamp_first_seen=first_seen.get(dest),
timestamp_last_seen=last_seen.get(dest),
risk_description=(
"A high total volume was sent to one external destination across "
"multiple connections, which may indicate gradual exfiltration."
),
recommendation="Review the destination and enforce egress filtering or DLP.",
evidence={"uploaded_bytes_total": total,
"dedup_key": f"aggup:{dest}"},
related_packets=sorted(set(packets_for_dest.get(dest, []))),
related_streams=sorted(set(streams_for_dest.get(dest, []))),
))
elif per_dest_count[dest] >= _MANY_SMALL_TRANSFERS:
findings.append(Finding(
title="Many small outbound transfers to one target",
category=Category.EXFILTRATION,
severity=Severity.MEDIUM, confidence=Confidence.LOW,
source_ip=src_for_dest.get(dest), destination_ip=dest,
risk_description=(
"Many small outbound connections to one target may indicate "
"segmented exfiltration or beaconing."
),
recommendation="Investigate the connection pattern and destination.",
evidence={"transfer_count": per_dest_count[dest],
"dedup_key": f"manyup:{dest}"},
related_packets=sorted(set(packets_for_dest.get(dest, []))),
related_streams=sorted(set(streams_for_dest.get(dest, []))),
))
# DNS volume is a weak signal; the DNS detector provides deeper evidence.
if len(ctx.dns_queries) >= _DNS_TXT_VOLUME:
findings.append(Finding(
title="Very high DNS query volume",
category=Category.EXFILTRATION,
severity=Severity.MEDIUM, confidence=Confidence.LOW,
protocol="dns",
risk_description=(
"Unusually high DNS volume may indicate DNS tunneling or exfiltration."
),
recommendation="Review DNS logs and correlated DNS anomaly findings.",
evidence={"dns_query_count": len(ctx.dns_queries), "dedup_key": "dns_volume"},
related_packets=sorted(
{q.packet_number for q in ctx.dns_queries if q.packet_number}
)[:8],
))
return self._tag(findings)
@@ -0,0 +1,107 @@
"""Detect insecure or unexpected legacy protocols."""
from __future__ import annotations
from collections import defaultdict
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
# Port to protocol name, severity, risk, and recommendation.
_RISKY_PORTS = {
21: ("FTP", Severity.HIGH, "FTP transmits data and credentials in clear text.",
"Replace FTP with FTPS or SFTP."),
23: ("Telnet", Severity.CRITICAL, "Telnet transmits all content in clear text.",
"Disable Telnet and use SSH."),
69: ("TFTP", Severity.HIGH, "TFTP has no authentication or encryption.",
"Avoid TFTP or restrict it to an isolated network."),
80: ("HTTP", Severity.MEDIUM, "HTTP content is unencrypted and observable.",
"Require HTTPS."),
110: ("POP3", Severity.HIGH, "POP3 without TLS exposes messages and passwords.",
"Use POP3S or STARTTLS."),
143: ("IMAP", Severity.HIGH, "IMAP without TLS exposes messages and passwords.",
"Use IMAPS or STARTTLS."),
161: ("SNMP", Severity.MEDIUM, "SNMP v1/v2c uses clear-text community strings.",
"Use SNMPv3 with authentication and privacy."),
389: ("LDAP", Severity.HIGH, "LDAP without TLS exposes directory credentials.",
"Use LDAPS or StartTLS."),
512: ("rexec", Severity.HIGH, "Remote shell services are insecure and unencrypted.",
"Disable the service and use SSH."),
513: ("rlogin", Severity.HIGH, "Remote login services are insecure and unencrypted.",
"Disable the service and use SSH."),
1433: ("MSSQL", Severity.MEDIUM, "The database service may be exposed without TLS.",
"Require TLS and restrict access."),
3306: ("MySQL", Severity.MEDIUM, "The database service may be exposed without TLS.",
"Require TLS and restrict access."),
6379: ("Redis", Severity.HIGH, "Redis may be exposed without authentication or encryption.",
"Enable authentication and restrict network exposure."),
11211: ("Memcached", Severity.HIGH, "Memcached can be abused for amplification.",
"Bind locally and restrict access."),
27017: ("MongoDB", Severity.HIGH, "MongoDB without authentication or TLS exposes data.",
"Enable authentication and TLS, then restrict access."),
}
@register
class InsecureProtocolDetector(BaseDetector):
name = "InsecureProtocolDetector"
description = "Detects clear-text and legacy protocols on known service ports."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
# Aggregate observed use of known risky service ports.
used: dict[int, dict] = defaultdict(lambda: {"src": None, "dst": None, "proto": None,
"first": None, "last": None, "packets": 0,
"pkts": [], "streams": set()})
for conv in ctx.conversations:
for port in (conv.dst_port, conv.src_port):
if port in _RISKY_PORTS:
e = used[port]
e["src"] = conv.src_ip
e["dst"] = conv.dst_ip
e["proto"] = conv.protocol
e["packets"] += conv.packets
if conv.packet_numbers:
e["pkts"].extend(conv.packet_numbers[:4])
if conv.stream_id is not None:
e["streams"].add(conv.stream_id)
if conv.first_seen is not None:
e["first"] = conv.first_seen if e["first"] is None else min(e["first"], conv.first_seen)
if conv.last_seen is not None:
e["last"] = conv.last_seen if e["last"] is None else max(e["last"], conv.last_seen)
for port, e in used.items():
name, sev, risk, rec = _RISKY_PORTS[port]
findings.append(Finding(
title=f"Insecure protocol: {name} (port {port})",
category=Category.INSECURE_PROTOCOL,
severity=sev, confidence=Confidence.HIGH,
source_ip=e["src"], destination_ip=e["dst"],
protocol=name.lower(), port=port,
timestamp_first_seen=e["first"], timestamp_last_seen=e["last"],
risk_description=risk,
recommendation=rec,
evidence={"port": port, "packets": e["packets"],
"dedup_key": f"insecure:{port}:{e['dst']}"},
related_packets=sorted(set(e["pkts"]))[:8],
related_streams=sorted(e["streams"])[:8],
))
smb1 = ctx.protocol_count("smb")
if smb1 and not ctx.protocol_count("smb2", "smb3"):
findings.append(Finding(
title="SMBv1 in use",
category=Category.INSECURE_PROTOCOL,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
protocol="smb",
risk_description=(
"SMBv1 is obsolete and vulnerable to well-known attacks such as EternalBlue."
),
recommendation="Disable SMBv1 and require SMBv2 or SMBv3.",
evidence={"smb1_packets": smb1, "dedup_key": "smbv1"},
))
return self._tag(findings)
@@ -0,0 +1,191 @@
"""Detect malware, command-and-control, and beaconing behavior."""
from __future__ import annotations
import statistics
from collections import defaultdict
from typing import Iterable, Optional
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
# Scanner and attack-tool user agents provide a strong signal.
_SCANNER_UA = (
"nikto", "masscan", "sqlmap", "nmap", "hydra", "zgrab", "nuclei",
"dirbuster", "gobuster", "wpscan", "acunetix", "nessus",
)
# Scripting clients are a weak signal because IoT automation may use them legitimately.
_SCRIPTING_UA = (
"curl", "wget", "python-requests", "powershell", "libwww",
)
# Common mobile and embedded clients are intentionally excluded to limit false positives.
_MIN_BEACONS = 5
_MAX_CV = 0.25
@register
class MalwareBeaconingDetector(BaseDetector):
name = "MalwareBeaconingDetector"
description = "Detects periodic external connections and suspicious user agents."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
findings.extend(self._detect_beaconing(ctx))
findings.extend(self._detect_suspicious_ua(ctx))
return self._tag(findings)
def _detect_beaconing(self, ctx: CaptureContext) -> list[Finding]:
"""Detect periodic traffic in long-lived flows and repeated short flows."""
findings: list[Finding] = []
reported: set[tuple] = set()
# First evaluate multiple timestamps captured within a single flow.
for conv in ctx.conversations:
ext_ip = _external_of(ctx, conv.src_ip, conv.dst_ip)
if not ext_ip:
continue
ts = sorted(t for t in conv.timestamps if t is not None)
if len(ts) < _MIN_BEACONS:
continue
internal = conv.dst_ip if ext_ip == conv.src_ip else conv.src_ip
port = conv.src_port if ext_ip == conv.src_ip else conv.dst_port
f = self._beacon_finding(
ts, src=internal, dst=ext_ip, protocol=conv.protocol, port=port,
streams=[conv.stream_id] if conv.stream_id is not None else [],
packets=list(conv.packet_numbers),
)
if f:
reported.add((internal, ext_ip, port))
findings.append(f)
# Then aggregate repeated short-lived connections to the same target.
events: dict[tuple, list[float]] = defaultdict(list)
meta: dict[tuple, dict] = {}
for conv in ctx.conversations:
if ctx.is_external(conv.dst_ip) and not ctx.is_external(conv.src_ip):
internal, external, port = conv.src_ip, conv.dst_ip, conv.dst_port
elif ctx.is_external(conv.src_ip) and not ctx.is_external(conv.dst_ip):
internal, external, port = conv.dst_ip, conv.src_ip, conv.src_port
else:
continue
if not (internal and external) or conv.first_seen is None:
continue
key = (internal, external, port)
events[key].append(conv.first_seen)
m = meta.setdefault(key, {"protocol": conv.protocol, "streams": set(), "packets": []})
if conv.stream_id is not None:
m["streams"].add(conv.stream_id)
# One representative packet per connection keeps evidence compact.
if conv.packet_numbers:
m["packets"].append(conv.packet_numbers[0])
for key, first_seens in events.items():
if key in reported or len(first_seens) < _MIN_BEACONS:
continue
internal, external, port = key
f = self._beacon_finding(
sorted(first_seens), src=internal, dst=external,
protocol=meta[key]["protocol"], port=port,
streams=sorted(meta[key]["streams"]),
packets=meta[key]["packets"],
connection_based=True,
)
if f:
findings.append(f)
return findings
def _beacon_finding(self, ts: list[float], *, src, dst, protocol, port,
streams, packets=None, connection_based: bool = False) -> Optional[Finding]:
intervals = [b - a for a, b in zip(ts, ts[1:]) if (b - a) > 0]
if len(intervals) < _MIN_BEACONS - 1:
return None
mean = statistics.mean(intervals)
if mean <= 0:
return None
cv = statistics.pstdev(intervals) / mean
if cv > _MAX_CV:
return None
conf = Confidence.MEDIUM if cv > 0.1 else Confidence.HIGH
evidence = {
"beacon_count": len(ts),
"interval_seconds_mean": round(mean, 2),
"interval_cv": round(cv, 3),
"signal": "distinct_connections" if connection_based else "single_flow",
"dedup_key": f"beacon:{src}:{dst}:{port}",
}
return Finding(
title="Periodic beaconing to an external target",
category=Category.MALWARE_BEACONING,
severity=Severity.HIGH, confidence=conf,
source_ip=src, destination_ip=dst,
protocol=protocol, port=port,
timestamp_first_seen=ts[0], timestamp_last_seen=ts[-1],
risk_description=(
"The device contacts an external target at highly regular intervals, "
"which is characteristic of command-and-control beaconing."
),
recommendation=(
"Check the target against threat intelligence, isolate the device, and "
"investigate it for compromise."
),
evidence=evidence,
related_packets=sorted({p for p in (packets or []) if p is not None}),
related_streams=[s for s in streams if s is not None],
)
def _detect_suspicious_ua(self, ctx: CaptureContext) -> list[Finding]:
findings: list[Finding] = []
seen_ua: set[str] = set()
for req in ctx.http_requests:
ua = (req.user_agent or "").lower()
if not ua:
continue
marker = next((m for m in _SCANNER_UA if m in ua), None)
if marker:
tier, sev, conf = "scanner", Severity.HIGH, Confidence.MEDIUM
risk = (
"The user agent belongs to a known scanning or attack tool and may "
"indicate active reconnaissance against the device."
)
else:
marker = next((m for m in _SCRIPTING_UA if m in ua), None)
if not marker:
continue
tier, sev, conf = "scripting", Severity.LOW, Confidence.LOW
risk = (
"An automated command-line client was used. This may be legitimate "
"automation, but it can also indicate scripts or malware."
)
if marker in seen_ua:
continue
seen_ua.add(marker)
findings.append(Finding(
title=f"Suspicious user agent: {marker}",
category=Category.MALWARE_BEACONING,
severity=sev, confidence=conf,
source_ip=req.src_ip, destination_ip=req.dst_ip,
protocol="http", port=req.dst_port,
timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
risk_description=risk,
recommendation="Verify that the source and client are authorized.",
evidence={"user_agent": req.user_agent, "host": req.host, "tier": tier,
"dedup_key": f"ua:{marker}:{req.src_ip}"},
related_packets=[p for p in [req.packet_number] if p],
))
return findings
def _external_of(ctx: CaptureContext, src_ip, dst_ip) -> Optional[str]:
"""Return the external address in a pair, if present."""
if ctx.is_external(dst_ip):
return dst_ip
if ctx.is_external(src_ip):
return src_ip
return None
+78
View File
@@ -0,0 +1,78 @@
"""Detect local network manipulation and man-in-the-middle indicators."""
from __future__ import annotations
from collections import defaultdict
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
@register
class MitmDetector(BaseDetector):
name = "MitmDetector"
description = "Detects ARP spoofing, duplicate IP mappings, and ARP floods."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
# Map each IP address to MAC addresses observed in ARP responses.
ip_to_macs: dict[str, set[str]] = defaultdict(set)
first_seen: dict[str, float] = {}
last_seen: dict[str, float] = {}
packets_for_ip: dict[str, list[int]] = defaultdict(list)
for ev in ctx.arp_events:
if ev.sender_ip and ev.sender_mac:
ip_to_macs[ev.sender_ip].add(ev.sender_mac)
if ev.timestamp is not None:
first_seen[ev.sender_ip] = min(first_seen.get(ev.sender_ip, ev.timestamp), ev.timestamp)
last_seen[ev.sender_ip] = max(last_seen.get(ev.sender_ip, ev.timestamp), ev.timestamp)
if ev.packet_number:
packets_for_ip[ev.sender_ip].append(ev.packet_number)
for ip, host in ctx.hosts.items():
if host.macs:
ip_to_macs[ip].update(host.macs)
for ip, macs in ip_to_macs.items():
if len(macs) > 1:
findings.append(Finding(
title="Multiple MAC addresses associated with one IP",
category=Category.MITM,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
source_ip=ip,
protocol="arp",
timestamp_first_seen=first_seen.get(ip),
timestamp_last_seen=last_seen.get(ip),
risk_description=(
"One IP address was advertised by multiple MAC addresses, a common "
"indicator of ARP spoofing or traffic interception."
),
recommendation=(
"Review ARP tables, enable Dynamic ARP Inspection and port security, "
"and identify the conflicting host."
),
evidence={"ip": ip, "macs": sorted(macs), "dedup_key": f"arpdup:{ip}"},
related_packets=sorted(packets_for_ip.get(ip, []))[:10],
))
# An ARP reply flood can accompany active cache poisoning.
grat = [e for e in ctx.arp_events if e.opcode == 2]
if len(grat) >= 50:
findings.append(Finding(
title="Unusually high number of ARP replies",
category=Category.MITM,
severity=Severity.MEDIUM, confidence=Confidence.LOW,
protocol="arp",
risk_description=(
"A large number of ARP replies may indicate active ARP cache poisoning."
),
recommendation="Inspect the reply source and investigate possible interception.",
evidence={"arp_reply_count": len(grat), "dedup_key": "arp_reply_flood"},
related_packets=sorted({e.packet_number for e in grat if e.packet_number})[:8],
))
return self._tag(findings)
@@ -0,0 +1,112 @@
"""Detect password hashes, challenge-response material, and Kerberos tickets."""
from __future__ import annotations
import re
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
_DIGEST_RE = re.compile(r"\bDigest\b", re.IGNORECASE)
@register
class PasswordHashDetector(BaseDetector):
name = "PasswordHashDetector"
description = (
"Detects NTLM, Kerberos, HTTP Digest, and wireless handshake material."
)
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
for am in ctx.auth_material:
kind = am.kind.lower()
if kind.startswith("kerberos"):
sev, conf = Severity.HIGH, Confidence.MEDIUM
title = "Kerberos authentication material in capture"
risk = (
"Kerberos AS-REP or TGS material may enable offline password attacks "
"when weak credentials are used."
)
rec = "Enforce pre-authentication, strong passwords, SPN hardening, and AES."
else:
sev, conf = Severity.HIGH, Confidence.MEDIUM
title = "NTLM authentication material in capture"
risk = (
"NTLM or NetNTLM material may enable offline cracking or relay attacks "
"without exposing a clear-text password."
)
rec = "Enable SMB signing and EPA, restrict NTLM, and prefer Kerberos."
evidence = {
"kind": am.kind, "account": am.account, "domain": am.domain,
"dedup_key": f"auth:{am.kind}:{am.src_ip}:{am.dst_ip}:{am.account}",
}
# Fully reconstructed hashcat material raises confidence.
if am.hashcat:
evidence["hashcat"] = am.hashcat
if am.hashcat_mode is not None:
evidence["hashcat_mode"] = am.hashcat_mode
if am.hash_format:
evidence["format"] = am.hash_format
conf = Confidence.HIGH
title += " (crackable)"
rec = (
"Rotate the affected password immediately. " + rec +
" The captured material is available in an offline-crackable format."
)
if am.etype:
evidence["etype"] = am.etype
findings.append(Finding(
title=title,
category=Category.PASSWORD_HASHES,
severity=sev, confidence=conf,
source_ip=am.src_ip, destination_ip=am.dst_ip, protocol=am.protocol,
timestamp_first_seen=am.timestamp, timestamp_last_seen=am.timestamp,
risk_description=risk, recommendation=rec,
evidence=evidence,
related_packets=[p for p in [am.packet_number] if p],
))
for req in ctx.http_requests:
if req.authorization and _DIGEST_RE.search(req.authorization):
findings.append(Finding(
title="HTTP Digest Authentication (Challenge-Response)",
category=Category.PASSWORD_HASHES,
severity=Severity.MEDIUM, confidence=Confidence.HIGH,
source_ip=req.src_ip, destination_ip=req.dst_ip,
protocol="http", port=req.dst_port,
timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
risk_description=(
"HTTP Digest transmits a response hash that may allow offline cracking "
"when weak passwords are used."
),
recommendation="Require HTTPS and enforce strong passwords.",
evidence={
"host": req.host, "uri": req.uri,
"dedup_key": f"digest:{req.src_ip}:{req.dst_ip}:{req.uri}",
},
related_packets=[p for p in [req.packet_number] if p],
))
# EAPOL frames indicate potentially crackable wireless handshakes.
eapol = ctx.protocol_count("eapol")
if eapol:
findings.append(Finding(
title="WPA/WPA2 handshake material (EAPOL) detected",
category=Category.PASSWORD_HASHES,
severity=Severity.HIGH, confidence=Confidence.MEDIUM,
protocol="eapol",
risk_description=(
"EAPOL frames from a four-way handshake may enable offline brute-force "
"attacks against the wireless password."
),
recommendation="Use a strong WPA2/WPA3 passphrase and prefer WPA3.",
evidence={"eapol_frames": eapol, "dedup_key": "eapol_handshake"},
))
return self._tag(findings)
@@ -0,0 +1,132 @@
"""Detect clear-text credentials."""
from __future__ import annotations
import base64
import re
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
# Common form fields used for credentials.
_USER_FIELDS = ("user", "username", "uname", "login", "email", "uid")
_PASS_FIELDS = ("pass", "passwd", "password", "pwd", "pw")
_FORM_RE = re.compile(r"([A-Za-z0-9_\-\.]+)=([^&\s]+)")
@register
class PlaintextCredentialDetector(BaseDetector):
name = "PlaintextCredentialDetector"
description = "Detects usernames and passwords transmitted without encryption."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
for pa in ctx.plaintext_auth:
if pa.username or pa.password:
findings.append(Finding(
title=f"Clear-text credentials over {pa.protocol.upper()}",
category=Category.PLAINTEXT_CREDENTIALS,
severity=Severity.CRITICAL,
confidence=Confidence.HIGH,
source_ip=pa.src_ip, destination_ip=pa.dst_ip,
protocol=pa.protocol, port=pa.dst_port,
timestamp_first_seen=pa.timestamp, timestamp_last_seen=pa.timestamp,
risk_description=(
"A username or password was transmitted without encryption. "
"A network observer could take over the account."
),
recommendation=(
f"Replace {pa.protocol.upper()} with an encrypted alternative "
"such as SFTP, IMAPS, POP3S, or SMTP with STARTTLS, then rotate "
"the affected password."
),
evidence={
"username": pa.username,
"password": _mask(pa.password),
"dedup_key": f"plain:{pa.protocol}:{pa.src_ip}:{pa.dst_ip}:{pa.username}",
},
related_packets=[p for p in [pa.packet_number] if p],
related_streams=[s for s in [pa.stream_id] if s],
))
for req in ctx.http_requests:
if req.is_tls:
continue
auth = (req.authorization or "").strip()
if auth.lower().startswith("basic "):
user, pwd = _decode_basic(auth[6:])
findings.append(Finding(
title="Clear-text HTTP Basic authentication",
category=Category.PLAINTEXT_CREDENTIALS,
severity=Severity.CRITICAL,
confidence=Confidence.HIGH,
source_ip=req.src_ip, destination_ip=req.dst_ip,
protocol="http", port=req.dst_port,
timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
risk_description=(
"HTTP Basic authentication sends credentials with Base64 encoding, "
"not encryption, over clear-text HTTP."
),
recommendation="Require HTTPS and rotate the affected password.",
evidence={
"host": req.host, "uri": req.uri,
"username": user, "password": _mask(pwd),
"dedup_key": f"basic:{req.src_ip}:{req.dst_ip}:{user}",
},
related_packets=[p for p in [req.packet_number] if p],
))
# Detect likely login forms in clear-text HTTP request bodies.
if req.method and req.method.upper() == "POST" and req.body:
fields = dict(_FORM_RE.findall(req.body))
lowered = {k.lower(): v for k, v in fields.items()}
has_user = any(k in lowered for k in _USER_FIELDS)
has_pass = any(k in lowered for k in _PASS_FIELDS)
if has_pass and (has_user or len(fields) <= 4):
user_val = next((lowered[k] for k in _USER_FIELDS if k in lowered), None)
pass_val = next((lowered[k] for k in _PASS_FIELDS if k in lowered), None)
findings.append(Finding(
title="Login form submitted over clear-text HTTP",
category=Category.PLAINTEXT_CREDENTIALS,
severity=Severity.CRITICAL,
confidence=Confidence.MEDIUM,
source_ip=req.src_ip, destination_ip=req.dst_ip,
protocol="http", port=req.dst_port,
timestamp_first_seen=req.timestamp, timestamp_last_seen=req.timestamp,
risk_description=(
"A login form was submitted in a clear-text HTTP request. "
"The credentials are visible to network observers."
),
recommendation="Serve the form only over HTTPS and rotate the password.",
evidence={
"host": req.host, "uri": req.uri,
"username": user_val, "password": _mask(pass_val),
"dedup_key": f"form:{req.src_ip}:{req.dst_ip}:{user_val}",
},
related_packets=[p for p in [req.packet_number] if p],
))
return self._tag(findings)
def _decode_basic(b64: str):
try:
raw = base64.b64decode(b64.strip() + "===").decode("utf-8", "ignore")
if ":" in raw:
u, p = raw.split(":", 1)
return u, p
return raw, None
except Exception:
return None, None
def _mask(secret):
if not secret:
return None
s = str(secret)
if len(s) <= 2:
return "*" * len(s)
return s[0] + "*" * (len(s) - 2) + s[-1]
@@ -0,0 +1,93 @@
"""Detect weak, obsolete, or suspicious TLS connections."""
from __future__ import annotations
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
_WEAK_VERSIONS = {"SSL 2.0", "SSL 3.0", "TLS 1.0", "TLS 1.1"}
# Weak cipher markers matched against normalized suite names.
_WEAK_CIPHER_MARKERS = ("RC4", "NULL", "EXPORT", "_DES_", "DES_CBC", "3DES", "MD5", "ANON")
@register
class TlsWeaknessDetector(BaseDetector):
name = "TlsWeaknessDetector"
description = "Detects obsolete TLS versions, weak ciphers, and certificate anomalies."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
seen: set[str] = set()
for s in ctx.tls_sessions:
target = s.sni or s.dst_ip
base_pkt = [p for p in [s.packet_number] if p]
if s.version and s.version in _WEAK_VERSIONS:
key = f"tlsver:{target}:{s.version}"
if key not in seen:
seen.add(key)
findings.append(Finding(
title=f"Obsolete TLS version: {s.version}",
category=Category.TLS_WEAKNESS,
severity=Severity.MEDIUM, confidence=Confidence.HIGH,
source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="tls", port=s.dst_port,
timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
risk_description=(
"An obsolete TLS or SSL version is in use and is vulnerable to "
"known protocol attacks."
),
recommendation="Require at least TLS 1.2 and prefer TLS 1.3.",
evidence={"version": s.version, "target": target, "dedup_key": key},
related_packets=base_pkt,
))
if s.cipher:
up = str(s.cipher).upper()
if any(m in up for m in _WEAK_CIPHER_MARKERS):
key = f"tlscipher:{target}:{s.cipher}"
if key not in seen:
seen.add(key)
findings.append(Finding(
title="Weak TLS cipher suite",
category=Category.TLS_WEAKNESS,
severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="tls", port=s.dst_port,
timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
risk_description=(
"A weak or obsolete cipher suite reduces transport protection."
),
recommendation="Allow only modern AEAD ciphers such as AES-GCM or ChaCha20.",
evidence={"cipher": s.cipher, "target": target, "dedup_key": key},
related_packets=base_pkt,
))
if s.cert_self_signed or s.cert_expired:
problem = []
if s.cert_self_signed:
problem.append("self-signed")
if s.cert_expired:
problem.append("expired")
key = f"tlscert:{target}:{'-'.join(problem)}"
if key not in seen:
seen.add(key)
findings.append(Finding(
title=f"Suspicious TLS certificate ({', '.join(problem)})",
category=Category.TLS_WEAKNESS,
severity=Severity.MEDIUM, confidence=Confidence.MEDIUM,
source_ip=s.src_ip, destination_ip=s.dst_ip, protocol="tls", port=s.dst_port,
timestamp_first_seen=s.timestamp, timestamp_last_seen=s.timestamp,
risk_description=(
"A self-signed or expired certificate may indicate endpoint "
"misconfiguration or TLS interception."
),
recommendation="Review the certificate chain and deploy a valid CA certificate.",
evidence={"subject_cn": s.cert_subject_cn, "issuer_cn": s.cert_issuer_cn,
"not_after": s.cert_not_after, "dedup_key": key},
related_packets=base_pkt,
))
return self._tag(findings)
@@ -0,0 +1,93 @@
"""Detect session cookies, bearer tokens, and API keys."""
from __future__ import annotations
import re
from typing import Iterable
from ..context import CaptureContext
from ..findings import Finding, Category, Severity, Confidence
from .base import BaseDetector, register
# JWTs consist of three base64url segments and typically start with ``eyJ``.
_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+")
_BEARER_RE = re.compile(r"\bBearer\s+([A-Za-z0-9_\-\.=]+)", re.IGNORECASE)
# Generic API key patterns in URLs, bodies, and headers.
_APIKEY_RE = re.compile(
r"(?i)(api[_\-]?key|apikey|access[_\-]?token|client[_\-]?secret|x[_\-]api[_\-]key)"
r"[=:\"']?\s*([A-Za-z0-9_\-]{16,})"
)
_SESSION_COOKIE_RE = re.compile(
r"(?i)\b(sessionid|session|sid|phpsessid|jsessionid|asp\.net_sessionid|auth|token)="
r"([A-Za-z0-9%_\-\.]{8,})"
)
@register
class TokenSecretDetector(BaseDetector):
name = "TokenSecretDetector"
description = "Detects JWTs, bearer tokens, API keys, and session cookies."
def detect(self, ctx: CaptureContext) -> Iterable[Finding]:
findings: list[Finding] = []
seen: set[str] = set()
for payload in ctx.text_payloads:
text = payload.get("text") or ""
over_http = payload.get("protocol") == "http"
for m in _JWT_RE.findall(text):
findings.append(self._mk(
payload, "Clear-text JWT (JSON Web Token)",
"A JWT was transmitted in clear text and may grant passwordless access "
"until it expires.",
"Revoke and rotate the token, shorten its lifetime, and require HTTPS.",
secret=m, conf=Confidence.HIGH, seen=seen, kind="jwt"))
for m in _BEARER_RE.findall(text):
findings.append(self._mk(
payload, "Clear-text bearer or OAuth token",
"A bearer token was transmitted and may grant access without a password.",
"Rotate the token, shorten its lifetime, and require HTTPS.",
secret=m, conf=Confidence.HIGH, seen=seen, kind="bearer"))
for _, m in _APIKEY_RE.findall(text):
findings.append(self._mk(
payload, "Clear-text API key or client secret",
"An API key or client secret was transmitted in clear text.",
"Rotate the secret, store it securely, and require HTTPS.",
secret=m, conf=Confidence.MEDIUM, seen=seen, kind="apikey"))
# Report session cookies only when the transport is unencrypted.
if over_http:
for _, m in _SESSION_COOKIE_RE.findall(text):
findings.append(self._mk(
payload, "Session cookie over clear-text HTTP",
"An unencrypted session cookie may enable session hijacking.",
"Set Secure and HttpOnly cookie flags and require HTTPS.",
secret=m, conf=Confidence.MEDIUM, seen=seen, kind="cookie"))
return self._tag([f for f in findings if f is not None])
def _mk(self, payload, title, risk, rec, *, secret, conf, seen, kind):
key = f"{kind}:{secret[:24]}"
if key in seen:
return None
seen.add(key)
return Finding(
title=title,
category=Category.TOKENS_SECRETS,
severity=Severity.HIGH, confidence=conf,
source_ip=payload.get("src_ip"), destination_ip=payload.get("dst_ip"),
protocol=payload.get("protocol"),
timestamp_first_seen=payload.get("timestamp"),
timestamp_last_seen=payload.get("timestamp"),
risk_description=risk, recommendation=rec,
evidence={"secret_preview": _preview(secret), "type": kind, "dedup_key": key},
related_packets=[p for p in [payload.get("packet_number")] if p],
)
def _preview(secret: str) -> str:
s = str(secret)
return s[:8] + "…" + s[-4:] if len(s) > 16 else s[:4] + "…"
+120
View File
@@ -0,0 +1,120 @@
"""Run all registered detectors and return a prioritized analysis result."""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Optional
from .context import CaptureContext
from .findings import Finding
from . import scoring
from . import provenance
from . import detectors as detectors_pkg
@dataclass
class AnalysisResult:
findings: list[Finding] = field(default_factory=list)
summary: dict = field(default_factory=dict)
meta: dict = field(default_factory=dict)
def to_dict(self) -> dict:
return {
"meta": self.meta,
"summary": _summary_to_dict(self.summary),
"findings": [f.to_dict() for f in self.findings],
}
class AnalysisEngine:
def __init__(self, detector_list=None):
# Use every registered detector unless a test supplies a subset.
self.detectors = detector_list if detector_list is not None else detectors_pkg.build_all()
def analyze_context(self, ctx: CaptureContext) -> AnalysisResult:
started = time.time()
raw: list[Finding] = []
errors: dict[str, str] = {}
for det in self.detectors:
try:
raw.extend(list(det.detect(ctx)))
except Exception as exc: # One detector must not abort the full analysis.
errors[det.name] = repr(exc)
deduped = scoring.dedupe(raw)
ordered = scoring.sort_by_priority(deduped)
# Attach packet-level provenance before the summary is generated.
capture_file = _basename(ctx.path)
for f in ordered:
_enrich_provenance(f, capture_file)
summary = scoring.summarize(ordered)
return AnalysisResult(
findings=ordered,
summary=summary,
meta={
"capture_path": ctx.path,
"packet_count": ctx.packet_count,
"duration_seconds": round(ctx.duration, 3),
"detectors_run": [d.name for d in self.detectors],
"detector_errors": errors,
"analysis_time_seconds": round(time.time() - started, 3),
"raw_finding_count": len(raw),
"finding_count": len(ordered),
},
)
def analyze_pcap(self, path: str, packet_limit: Optional[int] = None) -> AnalysisResult:
# Keep context-only analysis usable without pyshark or tshark.
from .extract_pyshark import build_context_from_pcap
ctx = build_context_from_pcap(path, packet_limit=packet_limit)
return self.analyze_context(ctx)
def result_from_dict(data: dict) -> AnalysisResult:
"""Reconstruct a result and recalculate its summary from stored findings."""
findings = [Finding.from_dict(f) for f in data.get("findings", [])]
summary = scoring.summarize(findings)
meta = data.get("meta", {})
return AnalysisResult(findings=findings, summary=summary, meta=meta)
def _summary_to_dict(summary: dict) -> dict:
out = dict(summary)
if "highlights" in out:
out["highlights"] = [f.to_dict() for f in out["highlights"]]
if "top" in out:
out["top"] = [f.to_dict() for f in out["top"]]
return out
def _basename(path) -> Optional[str]:
if not path:
return None
import os
return os.path.basename(str(path)) or str(path)
def _enrich_provenance(f: Finding, capture_file) -> None:
"""Add capture, artifact, and packet sources without replacing detector data."""
ev = f.evidence
if capture_file and not ev.get("capture_file"):
ev["capture_file"] = capture_file
if "artifact" not in ev:
art = provenance.derive_artifact(ev)
if art is not None:
ev["artifact"] = art
if "sources" not in ev:
ev["sources"] = provenance.build_sources(
related_packets=f.related_packets,
related_streams=f.related_streams,
src_ip=f.source_ip, dst_ip=f.destination_ip,
protocol=f.protocol, port=f.port,
timestamp_first_seen=f.timestamp_first_seen,
capture=capture_file,
)
# Preserve the complete evidence count when the displayed source list is bounded.
if f.related_packets and "source_packet_count" not in ev:
ev["source_packet_count"] = len(f.related_packets)
@@ -0,0 +1,11 @@
"""Optional, lazily loaded hash and endpoint enrichment."""
from .hash_crack import HashCrack, crack_hash
from .url_probe import UrlProbe, probe_url
from .engine import EnrichmentEngine
__all__ = [
"HashCrack", "crack_hash",
"UrlProbe", "probe_url",
"EnrichmentEngine",
]
+39
View File
@@ -0,0 +1,39 @@
"""Coordinate optional enrichment and progress callbacks."""
from __future__ import annotations
from typing import Callable, Optional, Sequence
from ..artifacts import HashArtifact, UrlArtifact
from .hash_crack import HashCrack, crack_hash
from .url_probe import UrlProbe, probe_url
# progress(done, total, item) -> None
ProgressCb = Optional[Callable[[int, int, object], None]]
class EnrichmentEngine:
"""Run hash and endpoint enrichment for artifact lists."""
def crack_hashes(self, artifacts: Sequence[HashArtifact], *,
progress: ProgressCb = None, **crack_kwargs) -> list[HashCrack]:
results: list[HashCrack] = []
total = len(artifacts)
for i, art in enumerate(artifacts, 1):
res = crack_hash(art, **crack_kwargs)
results.append(res)
if progress:
progress(i, total, res)
return results
def probe_urls(self, artifacts: Sequence[UrlArtifact], *,
progress: ProgressCb = None, active: bool = False,
**probe_kwargs) -> list[UrlProbe]:
results: list[UrlProbe] = []
total = len(artifacts)
for i, art in enumerate(artifacts, 1):
res = probe_url(art, active=active, **probe_kwargs)
results.append(res)
if progress:
progress(i, total, res)
return results
@@ -0,0 +1,144 @@
"""Run bounded offline hashcat attacks against supported capture artifacts."""
from __future__ import annotations
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import time
from dataclasses import dataclass, field
from typing import Callable, Optional, Sequence, Union
from ..artifacts import HashArtifact, identify_hash_mode
@dataclass
class HashCrack:
value: str
hashcat_mode: Optional[int] = None
hash_format: Optional[str] = None
cracked: bool = False
plaintext: Optional[str] = None
duration_seconds: Optional[float] = None
# pending | cracked | not_cracked | timeout | error | unsupported
state: str = "pending"
error: Optional[str] = None
def to_dict(self) -> dict:
return {
"value": self.value, "hashcat_mode": self.hashcat_mode,
"hash_format": self.hash_format, "cracked": self.cracked,
"plaintext": self.plaintext, "duration_seconds": self.duration_seconds,
"state": self.state, "error": self.error,
}
# Keep the bundled demo list first; authorized deployments may add a system list.
DEFAULT_WORDLISTS = (
str(Path(__file__).resolve().parents[3] / "demo_files" / "demo_wordlist.txt"),
"/usr/share/wordlists/rockyou.txt",
)
def build_crack_command(hashcat_path: str, mode: int, hash_file: str,
wordlists: Sequence[str], potfile: str) -> list[str]:
"""Build the hashcat dictionary-attack command."""
cmd = [hashcat_path, "-m", str(mode), "-a", "0", "-D", "1,2",
hash_file, *wordlists, "--potfile-path", potfile, "--force"]
return cmd
def build_show_command(hashcat_path: str, mode: int, hash_file: str,
potfile: str) -> list[str]:
"""Build the hashcat command used to read cracked results."""
return [hashcat_path, "-m", str(mode), "--show", "--potfile-path", potfile, hash_file]
def parse_show_output(stdout: str) -> Optional[str]:
"""Extract plaintext from ``hashcat --show`` output."""
if not stdout:
return None
for line in stdout.strip().splitlines():
if ":" in line:
return line.rsplit(":", 1)[1]
return None
def crack_hash(
artifact: Union[HashArtifact, str],
*,
wordlists: Sequence[str] = DEFAULT_WORDLISTS,
hashcat_path: str = "hashcat",
timeout: int = 300,
potfile: Optional[str] = None,
runner: Callable[..., subprocess.CompletedProcess] = subprocess.run,
which: Callable[[str], Optional[str]] = shutil.which,
) -> HashCrack:
"""Crack one artifact and convert runtime failures into a result state."""
if isinstance(artifact, HashArtifact):
value = artifact.value
mode = artifact.hashcat_mode
fmt = artifact.hash_format
else:
value = str(artifact)
mode = None
fmt = None
if mode is None:
mode, fmt = identify_hash_mode(value, hint_format=fmt)
result = HashCrack(value=value, hashcat_mode=mode, hash_format=fmt)
if mode is None:
result.state = "unsupported"
result.error = "No supported hashcat mode was detected"
return result
if which(hashcat_path) is None:
result.state = "error"
result.error = "hashcat is not installed or not available on PATH"
return result
available = [w for w in wordlists if os.path.exists(w)]
if not available:
result.state = "error"
result.error = "No wordlist is available"
return result
tmpdir = tempfile.mkdtemp(prefix="hashcrack_")
hash_file = os.path.join(tmpdir, "hash.txt")
pot = potfile or os.path.join(tmpdir, "hashcat.potfile")
try:
with open(hash_file, "w", encoding="utf-8") as fh:
fh.write(value.strip() + "\n")
start = time.time()
try:
runner(build_crack_command(hashcat_path, mode, hash_file, available, pot),
capture_output=True, text=True, timeout=timeout)
except subprocess.TimeoutExpired:
result.state = "timeout"
result.duration_seconds = round(time.time() - start, 2)
result.error = f"hashcat timed out after {timeout}s"
return result
show = runner(build_show_command(hashcat_path, mode, hash_file, pot),
capture_output=True, text=True, timeout=timeout)
result.duration_seconds = round(time.time() - start, 2)
plaintext = parse_show_output(getattr(show, "stdout", "") or "")
if plaintext is not None:
result.cracked = True
result.plaintext = plaintext
result.state = "cracked"
else:
result.state = "not_cracked"
return result
except Exception as exc: # Enrichment failures must not stop capture analysis.
result.state = "error"
result.error = str(exc)[:300]
return result
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
+143
View File
@@ -0,0 +1,143 @@
"""Optionally enrich observed endpoints with DNS, port, and TLS checks.
Passive capture evidence remains authoritative. Active checks are opt-in and
limited to a small port set because operators must have authorization to scan.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Callable, Optional, Sequence, Union
from urllib.parse import urlparse
from ..artifacts import UrlArtifact
DEFAULT_PORTS = (22, 80, 443, 8000, 8080)
@dataclass
class UrlProbe:
url: str
host: str
resolved_ips: list[str] = field(default_factory=list)
open_ports: list[int] = field(default_factory=list)
tls_versions: list[str] = field(default_factory=list)
cipher_suites: list[str] = field(default_factory=list)
certificate: Optional[str] = None
active: bool = False
error: Optional[str] = None
def to_dict(self) -> dict:
return {
"url": self.url, "host": self.host,
"resolved_ips": self.resolved_ips, "open_ports": self.open_ports,
"tls_versions": self.tls_versions, "cipher_suites": self.cipher_suites,
"certificate": self.certificate, "active": self.active, "error": self.error,
}
def host_of(url: str) -> str:
"""Reduce a URL or domain value to its hostname."""
s = str(url).strip()
if "://" not in s:
s = "//" + s
netloc = urlparse(s).netloc or ""
host = netloc.split("@")[-1].split(":")[0]
return host or str(url).strip()
def probe_url(
artifact: Union[UrlArtifact, str],
*,
active: bool = False,
ports: Sequence[int] = DEFAULT_PORTS,
dns_resolver: Optional[Callable[[str], list]] = None,
port_scanner: Optional[Callable[[str, Sequence[int]], list]] = None,
tls_scanner: Optional[Callable[[str], dict]] = None,
) -> UrlProbe:
"""Return passive observations and optionally perform authorized active checks."""
if isinstance(artifact, UrlArtifact):
url = artifact.value
probe = UrlProbe(url=url, host=host_of(url))
probe.open_ports = sorted(artifact.observed_ports)
probe.tls_versions = sorted(artifact.observed_tls)
else:
url = str(artifact)
probe = UrlProbe(url=url, host=host_of(url))
if not active:
return probe
probe.active = True
host = probe.host
try:
resolver = dns_resolver or _default_dns_resolver
probe.resolved_ips = resolver(host) or []
except Exception as exc:
probe.error = f"DNS: {exc}"
try:
scanner = port_scanner or _default_port_scanner
found = scanner(host, ports) or []
probe.open_ports = sorted(set(probe.open_ports) | set(found))
except Exception as exc:
probe.error = (probe.error or "") + f" Ports: {exc}"
try:
tls = tls_scanner or _default_tls_scanner
info = tls(host) or {}
if info.get("tls_version"):
probe.tls_versions = sorted(set(probe.tls_versions) | {info["tls_version"]})
if info.get("cipher"):
probe.cipher_suites = sorted(set(probe.cipher_suites) | {info["cipher"]})
probe.certificate = info.get("certificate")
except Exception as exc:
probe.error = (probe.error or "") + f" TLS: {exc}"
return probe
# --------------------------------------------------------------------------- #
# Lazy defaults are imported only when active checks are requested.
# --------------------------------------------------------------------------- #
def _default_dns_resolver(host: str) -> list[str]:
import dns.resolver # lazy
answers = dns.resolver.resolve(host, "A")
return [str(r) for r in answers]
def _default_port_scanner(host: str, ports: Sequence[int]) -> list[int]:
import nmap # lazy
nm = nmap.PortScanner()
port_arg = ",".join(str(p) for p in ports)
# TCP connect scanning works without raw-socket privileges; -Pn skips ICMP discovery.
nm.scan(hosts=host, arguments=f"-sT -Pn -p {port_arg} --max-retries 1 --host-timeout 20s")
open_ports: set[int] = set()
# nmap indexes results by resolved address, so iterate over every returned host.
for h in nm.all_hosts():
try:
for port in nm[h].all_tcp():
if nm[h]["tcp"][port]["state"] == "open":
open_ports.add(int(port))
except Exception:
continue
return sorted(open_ports)
def _default_tls_scanner(host: str) -> dict:
import socket
import ssl
ctx = ssl.create_default_context()
try:
with socket.create_connection((host, 443), timeout=5) as sock:
with ctx.wrap_socket(sock, server_hostname=host) as ssock:
cert = ssock.getpeercert()
cipher = ssock.cipher()
return {
"tls_version": ssock.version(),
"cipher": cipher[0] if cipher else None,
"certificate": "Present" if cert else "Not Present",
}
except Exception:
return {}
+763
View File
@@ -0,0 +1,763 @@
"""Build a normalized capture context with tshark through pyshark.
pyshark is loaded lazily, and individual packet handlers are isolated so one
unexpected packet cannot abort the complete analysis.
"""
from __future__ import annotations
import base64
import os
from typing import Optional
from .context import (
CaptureContext,
Conversation,
DnsQuery,
HttpRequest,
HttpResponse,
TlsSession,
ArpEvent,
PlaintextAuth,
AuthMaterial,
FileTransfer,
)
from . import crackformats
# Known clear-text protocols used by extraction heuristics.
CLEARTEXT_PORTS = {
21: "ftp",
23: "telnet",
25: "smtp",
80: "http",
110: "pop3",
143: "imap",
389: "ldap",
161: "snmp",
69: "tftp",
}
def build_context_from_pcap(path: str, packet_limit: Optional[int] = None) -> CaptureContext:
"""Read a capture and return its normalized context."""
if not os.path.exists(path):
raise FileNotFoundError(f"Capture not found: {path}")
try:
import pyshark # lazy import
except ImportError as exc: # pragma: no cover
raise RuntimeError(
"pyshark or tshark is unavailable. Install tshark or use the backend container."
) from exc
ctx = CaptureContext(path=path)
cap = pyshark.FileCapture(path, keep_packets=False)
conv_index: dict[tuple, Conversation] = {}
# Transient state correlates NTLM and SMTP authentication across packets.
state: dict = {"ntlm_chal": {}, "smtp_auth": {}}
n = 0
try:
for pkt in cap:
n += 1
if packet_limit and n > packet_limit:
break
try:
_handle_packet(ctx, pkt, conv_index, state)
except Exception:
# Skip isolated malformed or unexpected packets.
continue
except Exception as exc:
if exc.__class__.__name__ == "TSharkNotFoundException":
raise RuntimeError(
"tshark was not found. Install Wireshark/tshark or use the backend container."
) from exc
raise
finally:
try:
cap.close()
except Exception:
pass
ctx.packet_count = n
ctx.conversations = list(conv_index.values())
return ctx
# --------------------------------------------------------------------------- #
# Packet handlers
# --------------------------------------------------------------------------- #
def _ts(pkt) -> Optional[float]:
try:
return float(pkt.sniff_timestamp)
except Exception:
return None
def _num(pkt) -> Optional[int]:
try:
return int(pkt.number)
except Exception:
return None
def _handle_packet(ctx: CaptureContext, pkt, conv_index: dict, state: Optional[dict] = None) -> None:
if state is None:
state = {"ntlm_chal": {}, "smtp_auth": {}}
ts = _ts(pkt)
if ts is not None:
ctx.start_time = ts if ctx.start_time is None else min(ctx.start_time, ts)
ctx.end_time = ts if ctx.end_time is None else max(ctx.end_time, ts)
# Count packets by the highest decoded protocol layer.
try:
top = pkt.highest_layer
ctx.protocol_stats[top] = ctx.protocol_stats.get(top, 0) + 1
except Exception:
pass
layers = {l.layer_name for l in pkt.layers}
src_ip = dst_ip = None
length = 0
try:
length = int(pkt.length)
except Exception:
pass
if "ip" in layers:
try:
src_ip = pkt.ip.src
dst_ip = pkt.ip.dst
except Exception:
pass
elif "ipv6" in layers:
try:
src_ip = pkt.ipv6.src
dst_ip = pkt.ipv6.dst
except Exception:
pass
# MAC-Adressen / Hosts
src_mac = dst_mac = None
if "eth" in layers:
try:
src_mac, dst_mac = pkt.eth.src, pkt.eth.dst
except Exception:
pass
src_port = dst_port = None
transport = None
if "tcp" in layers:
transport = "tcp"
try:
src_port = int(pkt.tcp.srcport)
dst_port = int(pkt.tcp.dstport)
except Exception:
pass
elif "udp" in layers:
transport = "udp"
try:
src_port = int(pkt.udp.srcport)
dst_port = int(pkt.udp.dstport)
except Exception:
pass
if src_ip:
h = ctx.host(src_ip)
h.bytes_sent += length
if src_mac:
h.macs.add(src_mac)
if dst_port:
h.ports_contacted.add(dst_port)
if dst_ip:
h = ctx.host(dst_ip)
h.bytes_received += length
if dst_port:
h.ports_offered.add(dst_port)
# Conversation aggregieren
if src_ip and dst_ip and transport:
key = tuple(sorted([(src_ip, src_port), (dst_ip, dst_port)]) + [transport])
conv = conv_index.get(key)
if conv is None:
conv = Conversation(
src_ip=src_ip, dst_ip=dst_ip, src_port=src_port, dst_port=dst_port,
protocol=transport, first_seen=ts, last_seen=ts,
)
try:
conv.stream_id = int(pkt.tcp.stream) if transport == "tcp" else None
except Exception:
conv.stream_id = None
conv_index[key] = conv
conv.packets += 1
conv.bytes += length
pnum = _num(pkt)
if pnum is not None and len(conv.packet_numbers) < 20:
conv.packet_numbers.append(pnum)
if ts is not None:
conv.timestamps.append(ts)
conv.last_seen = ts if conv.last_seen is None else max(conv.last_seen, ts)
conv.first_seen = ts if conv.first_seen is None else min(conv.first_seen, ts)
if src_ip == conv.src_ip:
conv.bytes_src_to_dst += length
else:
conv.bytes_dst_to_src += length
# TCP stream IDs support cross-packet authentication correlation.
stream_id = None
if transport == "tcp":
try:
stream_id = int(pkt.tcp.stream)
except Exception:
stream_id = None
# Protocol-specific extraction.
if "arp" in layers:
_handle_arp(ctx, pkt, ts)
if "dns" in layers:
_handle_dns(ctx, pkt, src_ip, dst_ip, ts)
if "http" in layers:
_handle_http(ctx, pkt, src_ip, dst_ip, dst_port, ts)
if "tls" in layers or "ssl" in layers:
_handle_tls(ctx, pkt, src_ip, dst_ip, dst_port, ts)
if "ftp" in layers:
_handle_ftp(ctx, pkt, src_ip, dst_ip, dst_port, ts)
if "telnet" in layers:
_handle_telnet(ctx, pkt, src_ip, dst_ip, dst_port, ts)
if "pop" in layers:
_handle_pop3(ctx, pkt, src_ip, dst_ip, dst_port, ts)
if "imap" in layers:
_handle_imap(ctx, pkt, src_ip, dst_ip, dst_port, ts)
if "smtp" in layers:
_handle_smtp(ctx, pkt, src_ip, dst_ip, dst_port, ts, stream_id, state)
if "ntlmssp" in layers:
_handle_ntlm(ctx, pkt, src_ip, dst_ip, ts, stream_id, state)
if "kerberos" in layers:
_handle_kerberos(ctx, pkt, src_ip, dst_ip, ts)
# Generic payload extraction finds artifacts that protocol fields may omit.
_handle_raw_payload(ctx, pkt, layers, src_ip, dst_ip, ts)
def _handle_arp(ctx, pkt, ts):
try:
a = pkt.arp
ctx.arp_events.append(ArpEvent(
sender_ip=getattr(a, "src_proto_ipv4", None),
sender_mac=getattr(a, "src_hw_mac", None),
target_ip=getattr(a, "dst_proto_ipv4", None),
opcode=int(getattr(a, "opcode", 0) or 0),
timestamp=ts,
packet_number=_num(pkt),
))
except Exception:
pass
def _handle_dns(ctx, pkt, src_ip, dst_ip, ts):
try:
d = pkt.dns
# Ignore DNS records without a query name.
qname = getattr(d, "qry_name", None)
if not qname:
return
rcode = getattr(d, "flags_rcode", None)
rcode_map = {"0": "NOERROR", "3": "NXDOMAIN"}
response_ips = []
for attr in ("a", "aaaa"):
response_ips.extend(_all_field_values(d, attr))
ctx.dns_queries.append(DnsQuery(
query_name=qname,
qtype=_dns_qtype_name(getattr(d, "qry_type", None)),
response_ips=response_ips,
rcode=rcode_map.get(str(rcode), rcode),
timestamp=ts,
src_ip=src_ip,
dst_ip=dst_ip,
packet_number=_num(pkt),
))
except Exception:
pass
def _handle_http(ctx, pkt, src_ip, dst_ip, dst_port, ts):
try:
h = pkt.http
if hasattr(h, "request_method") or hasattr(h, "request_uri"):
req = HttpRequest(
method=getattr(h, "request_method", None),
host=getattr(h, "host", None),
uri=getattr(h, "request_uri", None),
user_agent=getattr(h, "user_agent", None),
authorization=getattr(h, "authorization", None),
cookies=getattr(h, "cookie", None),
body=getattr(h, "file_data", None),
src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port, timestamp=ts,
packet_number=_num(pkt),
)
ctx.http_requests.append(req)
# Preserve visible HTTP text for artifact detection.
for piece in (req.uri, req.cookies, req.authorization, req.body):
if piece:
ctx.text_payloads.append({
"text": str(piece), "protocol": "http",
"src_ip": src_ip, "dst_ip": dst_ip,
"packet_number": _num(pkt), "timestamp": ts,
})
if hasattr(h, "response_code"):
ct = getattr(h, "content_type", None)
ctx.http_responses.append(HttpResponse(
status_code=_safe_int(getattr(h, "response_code", None)),
content_type=ct,
server=getattr(h, "server", None),
src_ip=src_ip, dst_ip=dst_ip, timestamp=ts, packet_number=_num(pkt),
))
# Record only responses that look like file downloads.
disposition = getattr(h, "content_disposition", None)
filename = _filename_from_disposition(disposition)
if filename or _is_download_content_type(ct):
ext = os.path.splitext(filename or "")[1].lower() or None
ctx.file_transfers.append(FileTransfer(
filename=filename, extension=ext, content_type=ct,
protocol="http", src_ip=src_ip, dst_ip=dst_ip,
direction="download", timestamp=ts, packet_number=_num(pkt),
))
except Exception:
pass
def _handle_tls(ctx, pkt, src_ip, dst_ip, dst_port, ts):
try:
layer = getattr(pkt, "tls", None) or getattr(pkt, "ssl", None)
if layer is None:
return
# TLS 1.3 reports its real version in the supported-versions extension.
version = (
getattr(layer, "handshake_extensions_supported_version", None)
or getattr(layer, "handshake_version", None)
or getattr(layer, "record_version", None)
)
subject_cn = _first(
getattr(layer, "x509sat_printablestring", None),
getattr(layer, "x509ce_dnsname", None),
)
issuer_cn = getattr(layer, "x509sat_printablestring_1", None)
not_after = (
getattr(layer, "x509af_utctime", None)
or getattr(layer, "x509af_notafter", None)
or getattr(layer, "x509af_utctime_1", None)
)
ctx.tls_sessions.append(TlsSession(
src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port or 443,
version=_tls_version_name(version),
cipher=getattr(layer, "handshake_ciphersuite", None),
sni=getattr(layer, "handshake_extensions_server_name", None),
cert_subject_cn=subject_cn,
cert_issuer_cn=issuer_cn,
cert_not_after=not_after,
cert_self_signed=_cert_is_self_signed(subject_cn, issuer_cn),
cert_expired=_cert_is_expired(not_after, ctx.end_time or ts),
timestamp=ts, packet_number=_num(pkt),
))
except Exception:
pass
def _handle_ftp(ctx, pkt, src_ip, dst_ip, dst_port, ts):
try:
f = pkt.ftp
cmd = getattr(f, "request_command", None)
arg = getattr(f, "request_arg", None)
if cmd and cmd.upper() in ("USER", "PASS"):
pa = _last_plaintext(ctx, "ftp", src_ip, dst_ip, dst_port)
if cmd.upper() == "USER":
pa.username = arg
else:
pa.password = arg
pa.timestamp = ts
pa.packet_number = _num(pkt)
except Exception:
pass
def _handle_telnet(ctx, pkt, src_ip, dst_ip, dst_port, ts):
try:
data = getattr(pkt.telnet, "data", None)
if data:
ctx.text_payloads.append({
"text": str(data), "protocol": "telnet",
"src_ip": src_ip, "dst_ip": dst_ip,
"packet_number": _num(pkt), "timestamp": ts,
})
except Exception:
pass
def _handle_pop3(ctx, pkt, src_ip, dst_ip, dst_port, ts):
"""POP3 USER/PASS im Klartext."""
try:
p = pkt.pop
cmd = (getattr(p, "request_command", None) or "").upper()
param = getattr(p, "request_parameter", None)
if cmd in ("USER", "PASS") and param:
pa = _last_plaintext(ctx, "pop3", src_ip, dst_ip, dst_port)
if cmd == "USER":
pa.username = param
else:
pa.password = param
pa.timestamp = ts
pa.packet_number = _num(pkt)
except Exception:
pass
def _handle_imap(ctx, pkt, src_ip, dst_ip, dst_port, ts):
"""Extract clear-text IMAP LOGIN credentials."""
try:
i = pkt.imap
line = getattr(i, "request", None) or getattr(i, "request_command", None)
if not line:
return
parts = str(line).split()
# Accept both tagged and untagged IMAP LOGIN forms.
idx = next((k for k, tok in enumerate(parts) if tok.upper() == "LOGIN"), None)
if idx is not None and len(parts) >= idx + 3:
user = parts[idx + 1].strip('"')
pw = parts[idx + 2].strip('"')
pa = _last_plaintext(ctx, "imap", src_ip, dst_ip, dst_port)
pa.username, pa.password = user, pw
pa.timestamp = ts
pa.packet_number = _num(pkt)
except Exception:
pass
def _handle_smtp(ctx, pkt, src_ip, dst_ip, dst_port, ts, stream_id, state):
"""Extract Base64-encoded SMTP AUTH LOGIN or PLAIN credentials."""
try:
s = pkt.smtp
param = getattr(s, "req_parameter", None) or getattr(s, "request_parameter", None)
cmd = (getattr(s, "req_command", None) or getattr(s, "request_command", None) or "").upper()
if not param and not cmd:
return
pending = state["smtp_auth"].setdefault(stream_id, {"mech": None, "tokens": []})
text = f"{cmd} {param}".strip()
upper = text.upper()
if "AUTH LOGIN" in upper:
pending["mech"] = "login"
return
if "AUTH PLAIN" in upper:
pending["mech"] = "plain"
# AUTH PLAIN may carry its token on the command line.
tail = text.split()
if len(tail) >= 3:
creds = crackformats.decode_smtp_auth_plain(tail[2])
_emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt)
state["smtp_auth"].pop(stream_id, None)
return
# AUTH LOGIN sends username and password in subsequent lines.
if pending["mech"] == "login" and param:
pending["tokens"].append(param)
if len(pending["tokens"]) >= 2:
user, pw = crackformats.decode_smtp_auth_login(
pending["tokens"][0], pending["tokens"][1])
_emit_smtp(ctx, (user, pw), src_ip, dst_ip, dst_port, ts, pkt)
state["smtp_auth"].pop(stream_id, None)
elif pending["mech"] == "plain" and param:
creds = crackformats.decode_smtp_auth_plain(param)
_emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt)
state["smtp_auth"].pop(stream_id, None)
except Exception:
pass
def _emit_smtp(ctx, creds, src_ip, dst_ip, dst_port, ts, pkt):
if not creds:
return
user, pw = creds
if not (user or pw):
return
ctx.plaintext_auth.append(PlaintextAuth(
protocol="smtp", username=user, password=pw,
src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port,
timestamp=ts, packet_number=_num(pkt),
))
def _handle_ntlm(ctx, pkt, src_ip, dst_ip, ts, stream_id, state):
"""Correlate NTLM challenge and response packets into hashcat material."""
try:
n = pkt.ntlmssp
msg_type = str(getattr(n, "messagetype", "") or "").lower()
# Retain the type-2 server challenge for the matching type-3 response.
server_chal = getattr(n, "ntlmserverchallenge", None)
if server_chal and ("0x00000002" in msg_type or msg_type.endswith("2")):
state["ntlm_chal"][stream_id] = str(server_chal)
# A type-3 message provides the account and NT response.
account = getattr(n, "auth_username", None)
domain = getattr(n, "auth_domain", None)
nt_resp = (getattr(n, "ntlmv2_response", None)
or getattr(n, "auth_ntresponse", None))
if account or nt_resp:
chal = state["ntlm_chal"].get(stream_id)
am = AuthMaterial(
kind="ntlmssp", account=account, domain=domain,
src_ip=src_ip, dst_ip=dst_ip, protocol="ntlm",
timestamp=ts, packet_number=_num(pkt),
server_challenge=chal, nt_response=str(nt_resp) if nt_resp else None,
)
built = crackformats.build_netntlmv2(account, domain, chal, nt_resp)
if built:
am.kind = "netntlmv2"
am.hashcat = built["hashcat"]
am.hashcat_mode = built["hashcat_mode"]
am.hash_format = built["format"]
ctx.auth_material.append(am)
except Exception:
pass
# Bound retained payload text for large captures.
_MAX_TEXT_PAYLOADS = 20000
def _handle_raw_payload(ctx, pkt, layers, src_ip, dst_ip, ts):
"""Retain bounded TCP or UDP text for generic artifact detection."""
if len(ctx.text_payloads) >= _MAX_TEXT_PAYLOADS:
return
raw = None
try:
if "tcp" in layers and hasattr(pkt.tcp, "payload"):
raw = pkt.tcp.payload
elif "udp" in layers and hasattr(pkt.udp, "payload"):
raw = pkt.udp.payload
elif "data" in layers:
raw = getattr(pkt.data, "data", None)
except Exception:
raw = None
if not raw:
return
text = _bytes_from_hexish(str(raw))
if not text or not text.strip():
return
try:
proto = (pkt.highest_layer or "raw").lower()
except Exception:
proto = "raw"
ctx.text_payloads.append({
"text": text[:16384], "protocol": proto,
"src_ip": src_ip, "dst_ip": dst_ip,
"packet_number": _num(pkt), "timestamp": ts,
})
def _bytes_from_hexish(raw: str) -> Optional[str]:
"""Convert a hex-formatted pyshark payload to text when possible."""
s = str(raw).strip()
compact = s.replace(":", "").replace(".", "").replace(" ", "")
if compact and len(compact) % 2 == 0 and all(c in "0123456789abcdefABCDEF" for c in compact):
try:
return bytes.fromhex(compact).decode("utf-8", "ignore")
except Exception:
return None
# Some dissectors already expose decoded text.
return s
def _handle_kerberos(ctx, pkt, src_ip, dst_ip, ts):
"""Extract Kerberos AS-REP and TGS-REP material for offline analysis."""
try:
k = pkt.kerberos
msg_type = str(getattr(k, "msg_type", "") or "")
account = getattr(k, "cnamestring", None) or getattr(k, "cnamestring_1", None)
realm = getattr(k, "realm", None) or getattr(k, "crealm", None)
etype = getattr(k, "etype", None)
cipher = getattr(k, "cipher", None) or getattr(k, "enc_part_cipher", None)
am = AuthMaterial(
kind=f"kerberos_{msg_type}" if msg_type else "kerberos",
account=account, domain=realm,
src_ip=src_ip, dst_ip=dst_ip, protocol="kerberos",
timestamp=ts, packet_number=_num(pkt),
etype=str(etype) if etype is not None else None,
)
# Kerberos message types 11 and 13 are AS-REP and TGS-REP.
built = None
if msg_type == "11":
built = crackformats.build_krb5_asrep(account, realm, etype, cipher)
elif msg_type == "13":
spn = getattr(k, "snamestring", None)
built = crackformats.build_krb5_tgsrep(account, realm, spn, etype, cipher)
if built:
am.hashcat = built["hashcat"]
am.hashcat_mode = built["hashcat_mode"]
am.hash_format = built["format"]
ctx.auth_material.append(am)
except Exception:
pass
# --------------------------------------------------------------------------- #
# kleine Helfer
# --------------------------------------------------------------------------- #
def _last_plaintext(ctx, protocol, src_ip, dst_ip, dst_port) -> PlaintextAuth:
"""Find or create the authentication record that combines USER and PASS."""
for pa in reversed(ctx.plaintext_auth):
if pa.protocol == protocol and pa.src_ip == src_ip and pa.dst_ip == dst_ip:
if pa.password is None or pa.username is None:
return pa
pa = PlaintextAuth(protocol=protocol, src_ip=src_ip, dst_ip=dst_ip, dst_port=dst_port)
ctx.plaintext_auth.append(pa)
return pa
def _safe_int(v):
try:
return int(v)
except Exception:
return None
def _tls_version_name(raw) -> Optional[str]:
mapping = {
"0x0300": "SSL 3.0", "0x0301": "TLS 1.0", "0x0302": "TLS 1.1",
"0x0303": "TLS 1.2", "0x0304": "TLS 1.3",
"768": "SSL 3.0", "769": "TLS 1.0", "770": "TLS 1.1",
"771": "TLS 1.2", "772": "TLS 1.3",
}
if raw is None:
return None
return mapping.get(str(raw).lower(), str(raw))
def _first(*vals):
"""Return the first non-empty dissector field value."""
for v in vals:
if v:
return v
return None
def _all_field_values(layer, name) -> list[str]:
"""Return every occurrence of a repeated pyshark field."""
out: list[str] = []
try:
field = getattr(layer, name, None)
if field is None:
return out
allf = getattr(field, "all_fields", None)
if allf:
for f in allf:
v = getattr(f, "show", None) or getattr(f, "showname_value", None)
if v:
out.append(str(v))
else:
out.append(str(field))
except Exception:
pass
return out
# tshark often exposes DNS query types as numeric values.
_DNS_QTYPES = {
"1": "A", "2": "NS", "5": "CNAME", "6": "SOA", "12": "PTR", "15": "MX",
"16": "TXT", "28": "AAAA", "33": "SRV", "255": "ANY", "10": "NULL",
"43": "DS", "48": "DNSKEY", "257": "CAA", "65": "HTTPS", "64": "SVCB",
}
def _dns_qtype_name(raw) -> Optional[str]:
if raw is None:
return None
s = str(raw).strip()
return _DNS_QTYPES.get(s, s.upper())
# Content types that indicate a file rather than ordinary web content.
_DOWNLOAD_CT_MARKERS = (
"application/octet-stream", "application/x-msdownload", "application/x-dosexec",
"application/x-executable", "application/vnd.microsoft.portable-executable",
"application/zip", "application/x-rar", "application/x-7z", "application/x-tar",
"application/gzip", "application/x-msi", "application/x-sh",
"application/java-archive", "application/x-elf",
)
def _is_download_content_type(ct) -> bool:
if not ct:
return False
low = str(ct).lower()
return any(m in low for m in _DOWNLOAD_CT_MARKERS)
def _filename_from_disposition(disposition) -> Optional[str]:
"""Extract a filename from a Content-Disposition header."""
if not disposition:
return None
import re as _re
m = _re.search(r'filename\*?=(?:"([^"]+)"|([^;]+))', str(disposition), _re.IGNORECASE)
if not m:
return None
name = (m.group(1) or m.group(2) or "").strip().strip('"')
# Remove an optional RFC 5987 character-set prefix.
if "''" in name:
name = name.split("''", 1)[1]
return name or None
def _cert_is_self_signed(subject_cn, issuer_cn) -> Optional[bool]:
"""Return whether subject and issuer match, or None if data is incomplete."""
if not subject_cn or not issuer_cn:
return None
return str(subject_cn).strip().lower() == str(issuer_cn).strip().lower()
def _cert_is_expired(not_after, now_epoch) -> Optional[bool]:
"""Compare a parsed certificate expiry with the capture timestamp."""
if not not_after or now_epoch is None:
return None
exp = _parse_cert_time(not_after)
if exp is None:
return None
try:
return float(now_epoch) > exp
except (TypeError, ValueError):
return None
def _parse_cert_time(raw) -> Optional[float]:
"""Convert common X.509 time formats to a UTC epoch timestamp."""
import datetime as _dt
s = str(raw).strip()
# tshark may already provide an epoch value.
try:
return float(s)
except ValueError:
pass
fmts = (
"%Y-%m-%d %H:%M:%S", # 2020-01-01 00:00:00
"%b %d %H:%M:%S %Y GMT", # Jan 1 00:00:00 2020 GMT
"%y%m%d%H%M%SZ", # UTCTime 200101000000Z
"%Y%m%d%H%M%SZ", # GeneralizedTime
"%Y-%m-%dT%H:%M:%SZ", # ISO 8601
)
for fmt in fmts:
try:
dt = _dt.datetime.strptime(s, fmt).replace(tzinfo=_dt.timezone.utc)
return dt.timestamp()
except ValueError:
continue
return None
+166
View File
@@ -0,0 +1,166 @@
"""Shared finding taxonomy for IoT PreTester 4.6 Demo.
The module deliberately has no Django or third-party dependencies so the same
model works in the web backend, CLI, and unit tests.
"""
from __future__ import annotations
import enum
import hashlib
from dataclasses import dataclass, field, asdict
from typing import Any, Optional
class Severity(enum.Enum):
"""Potential impact if the finding is valid."""
CRITICAL = "critical"
HIGH = "high"
MEDIUM = "medium"
LOW = "low"
INFO = "info"
@property
def rank(self) -> int:
return _SEVERITY_RANK[self]
class Confidence(enum.Enum):
"""Confidence in the evidence extracted from the capture."""
HIGH = "high"
MEDIUM = "medium"
LOW = "low"
@property
def rank(self) -> int:
return _CONFIDENCE_RANK[self]
class Category(enum.Enum):
"""Security finding categories covered by the detector set."""
PLAINTEXT_CREDENTIALS = "plaintext_credentials"
PASSWORD_HASHES = "password_hashes"
TOKENS_SECRETS = "tokens_secrets"
MALWARE_BEACONING = "malware_beaconing"
EXFILTRATION = "exfiltration"
MITM = "mitm"
INSECURE_PROTOCOL = "insecure_protocol"
TLS_WEAKNESS = "tls_weakness"
DNS_ANOMALY = "dns_anomaly"
BRUTEFORCE_SCAN_FILETRANSFER = "bruteforce_scan_filetransfer"
_SEVERITY_RANK = {
Severity.CRITICAL: 4,
Severity.HIGH: 3,
Severity.MEDIUM: 2,
Severity.LOW: 1,
Severity.INFO: 0,
}
_CONFIDENCE_RANK = {
Confidence.HIGH: 2,
Confidence.MEDIUM: 1,
Confidence.LOW: 0,
}
def _coerce(value, enum_cls):
"""Accept enum members and their serialized string values."""
if isinstance(value, enum_cls):
return value
if isinstance(value, str):
return enum_cls(value)
raise TypeError(f"{value!r} is not a valid {enum_cls.__name__}")
@dataclass
class Finding:
"""Shared data model for detector output and packet evidence."""
title: str
category: Category
severity: Severity
confidence: Confidence
risk_description: str
recommendation: str
affected_device: Optional[str] = None
source_ip: Optional[str] = None
destination_ip: Optional[str] = None
protocol: Optional[str] = None
port: Optional[int] = None
timestamp_first_seen: Optional[float] = None
timestamp_last_seen: Optional[float] = None
# Structured evidence captured by the detector.
evidence: dict[str, Any] = field(default_factory=dict)
related_packets: list[int] = field(default_factory=list)
related_streams: list[int] = field(default_factory=list)
# Filled by the engine with the detector class name.
detector: Optional[str] = None
finding_id: Optional[str] = None
def __post_init__(self) -> None:
self.category = _coerce(self.category, Category)
self.severity = _coerce(self.severity, Severity)
self.confidence = _coerce(self.confidence, Confidence)
if self.finding_id is None:
self.finding_id = self.compute_id()
def compute_id(self) -> str:
"""Build a stable identifier from the finding's identity fields."""
basis = "|".join(
str(x)
for x in (
self.category.value,
self.title,
self.source_ip,
self.destination_ip,
self.protocol,
self.port,
_evidence_fingerprint(self.evidence),
)
)
return hashlib.sha1(basis.encode("utf-8")).hexdigest()[:16]
@property
def priority(self) -> tuple[int, int]:
"""Sort by severity first, then confidence."""
return (self.severity.rank, self.confidence.rank)
def to_dict(self) -> dict[str, Any]:
data = asdict(self)
data["category"] = self.category.value
data["severity"] = self.severity.value
data["confidence"] = self.confidence.value
return data
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "Finding":
"""Reconstruct a finding from serialized data."""
fields = {
"title", "category", "severity", "confidence", "risk_description",
"recommendation", "affected_device", "source_ip", "destination_ip",
"protocol", "port", "timestamp_first_seen", "timestamp_last_seen",
"evidence", "related_packets", "related_streams", "detector", "finding_id",
}
kwargs = {k: v for k, v in data.items() if k in fields}
return cls(**kwargs)
def _evidence_fingerprint(evidence: dict[str, Any]) -> str:
"""Prefer an explicit deduplication key, then stable sorted evidence."""
if not evidence:
return ""
if "dedup_key" in evidence:
return str(evidence["dedup_key"])
try:
items = sorted((str(k), str(v)) for k, v in evidence.items())
except Exception:
return str(evidence)
return ";".join(f"{k}={v}" for k, v in items)
+110
View File
@@ -0,0 +1,110 @@
"""Build packet-level provenance without coupling findings to PCAP I/O."""
from __future__ import annotations
from typing import Any, Optional
# Keep reports compact while preserving the complete packet count separately.
MAX_SOURCE_PACKETS = 8
def source_ref(*, packet: Optional[int] = None, stream: Optional[int] = None,
timestamp: Optional[float] = None, src_ip: Optional[str] = None,
dst_ip: Optional[str] = None, protocol: Optional[str] = None,
port: Optional[int] = None, capture: Optional[str] = None,
note: Optional[str] = None) -> dict:
"""Build a normalized source reference while omitting empty fields."""
ref: dict[str, Any] = {}
if packet is not None:
ref["packet"] = packet
if stream is not None:
ref["stream"] = stream
if timestamp is not None:
ref["timestamp"] = timestamp
if src_ip:
ref["src_ip"] = src_ip
if dst_ip:
ref["dst_ip"] = dst_ip
if protocol:
ref["protocol"] = protocol
if port:
ref["port"] = port
if capture:
ref["capture"] = capture
if note:
ref["note"] = note
return ref
# Priority-ordered evidence keys used to select the primary artifact.
_ARTIFACT_KEYS = [
("hashcat", "Hash (hashcat)"),
("secret_preview", "Secret/Token"),
("query_name", "DNS query"),
("cipher", "TLS-Cipher"),
("version", "TLS-Version"),
("user_agent", "User-Agent"),
("filename", "File"),
("uri", "URI"),
("subject_cn", "Certificate CN"),
("registered_domain", "Domain"),
("macs", "MAC addresses"),
]
def derive_artifact(evidence: dict) -> Optional[dict]:
"""Derive the most representative artifact from structured evidence."""
if not evidence:
return None
# Treat a username and masked password as one artifact.
user = evidence.get("username")
pw = evidence.get("password")
if user or pw:
return {"label": "Credentials", "value": f"{user or '?'} : {pw or '?'}"}
for key, label in _ARTIFACT_KEYS:
val = evidence.get(key)
if val:
if isinstance(val, (list, tuple)):
val = ", ".join(str(x) for x in val)
return {"label": label, "value": str(val)}
return None
def build_sources(*, related_packets: list[int], related_streams: list[int],
src_ip: Optional[str], dst_ip: Optional[str],
protocol: Optional[str], port: Optional[int],
timestamp_first_seen: Optional[float],
capture: Optional[str],
max_packets: int = MAX_SOURCE_PACKETS) -> list[dict]:
"""Build a bounded source list from packet, stream, and endpoint fields."""
packets = list(dict.fromkeys(p for p in (related_packets or []) if p is not None))
streams = list(dict.fromkeys(s for s in (related_streams or []) if s is not None))
single_stream = streams[0] if len(streams) == 1 else None
sources: list[dict] = []
if packets:
single_ts = timestamp_first_seen if len(packets) == 1 else None
for p in packets[:max_packets]:
sources.append(source_ref(
packet=p, stream=single_stream, timestamp=single_ts,
src_ip=src_ip, dst_ip=dst_ip, protocol=protocol, port=port,
capture=capture,
))
elif streams:
for s in streams[:max_packets]:
sources.append(source_ref(
stream=s, src_ip=src_ip, dst_ip=dst_ip, protocol=protocol,
port=port, capture=capture,
))
elif src_ip or dst_ip or timestamp_first_seen is not None:
sources.append(source_ref(
src_ip=src_ip, dst_ip=dst_ip, protocol=protocol, port=port,
timestamp=timestamp_first_seen, capture=capture,
note="aggregated evidence; no single packet is available",
))
elif capture:
sources.append(source_ref(capture=capture,
note="aggregated evidence; no single packet is available"))
return sources
@@ -0,0 +1,25 @@
"""Report renderers for Markdown, plain text, and JSON."""
from __future__ import annotations
from ..engine import AnalysisResult
from . import markdown as _md
from . import text as _txt
from . import json_report as _json
FORMATS = ("md", "txt", "json")
def render(result: AnalysisResult, fmt: str, extra: dict | None = None) -> str:
"""Render a report with optional endpoint and hash enrichment sections."""
fmt = (fmt or "md").lower()
if fmt in ("md", "markdown"):
return _md.render(result, extra)
if fmt in ("txt", "text"):
return _txt.render(result, extra)
if fmt == "json":
return _json.render(result, extra)
raise ValueError(f"Unknown report format: {fmt!r} (allowed: {', '.join(FORMATS)})")
__all__ = ["render", "FORMATS"]
@@ -0,0 +1,107 @@
"""Shared report-rendering helpers."""
from __future__ import annotations
import datetime as _dt
from ..findings import Finding, Severity
SEVERITY_LABEL = {
Severity.CRITICAL: "CRITICAL",
Severity.HIGH: "HIGH",
Severity.MEDIUM: "MEDIUM",
Severity.LOW: "LOW",
Severity.INFO: "INFO",
}
def fmt_ts(ts) -> str:
if ts is None:
return "—"
try:
return _dt.datetime.fromtimestamp(float(ts), _dt.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
except Exception:
return str(ts)
def affected_line(f: Finding) -> str:
"""Format affected devices, endpoints, and ports as one compact line."""
parts = []
if f.affected_device:
parts.append(f"Device {f.affected_device}")
if f.source_ip:
parts.append(f"Source {f.source_ip}")
if f.destination_ip:
parts.append(f"Destination {f.destination_ip}")
if f.protocol:
proto = f.protocol + (f":{f.port}" if f.port else "")
parts.append(proto)
for k in ("host", "registered_domain", "query_name", "target"):
if f.evidence.get(k):
parts.append(str(f.evidence[k]))
break
return ", ".join(parts) if parts else "—"
# Evidence rendered in dedicated sections is excluded from generic details.
_EVIDENCE_SKIP = {
"dedup_key", "sources", "artifact", "capture_file", "source_packet_count",
"occurrences",
}
def artifact_line(f: Finding) -> str:
"""Format the primary artifact as one compact line."""
art = f.evidence.get("artifact")
if isinstance(art, dict) and art.get("value"):
label = art.get("label")
return f"{label}: {art['value']}" if label else str(art["value"])
if art:
return str(art)
return "—"
def fmt_source(ref: dict) -> str:
"""Format one packet-level source reference."""
parts = []
if ref.get("packet") is not None:
parts.append(f"Packet #{ref['packet']}")
if ref.get("stream") is not None:
parts.append(f"Stream {ref['stream']}")
if ref.get("timestamp") is not None:
parts.append(fmt_ts(ref["timestamp"]))
if ref.get("src_ip") or ref.get("dst_ip"):
parts.append(f"{ref.get('src_ip', '?')} → {ref.get('dst_ip', '?')}")
if ref.get("protocol"):
proto = ref["protocol"] + (f":{ref['port']}" if ref.get("port") else "")
parts.append(proto)
if ref.get("capture"):
parts.append(f"Capture {ref['capture']}")
if ref.get("note"):
parts.append(str(ref["note"]))
return " · ".join(parts) if parts else "—"
def sources_list(f: Finding) -> list[str]:
"""Return formatted packet-level source references for a finding."""
refs = f.evidence.get("sources") or []
lines = [fmt_source(r) for r in refs]
total = f.evidence.get("source_packet_count")
if total and len(f.related_packets or []) and total > len(refs):
lines.append(f"… and {total - len(refs)} more packets")
return lines
def evidence_line(f: Finding) -> str:
"""Format packet, stream, time, and detector-specific evidence."""
parts = []
if f.related_packets:
pk = ", ".join(map(str, f.related_packets[:8]))
parts.append(f"Packets: {pk}")
if f.related_streams:
parts.append(f"Streams: {', '.join(map(str, f.related_streams[:8]))}")
parts.append(f"Time: {fmt_ts(f.timestamp_first_seen)} – {fmt_ts(f.timestamp_last_seen)}")
extras = {k: v for k, v in f.evidence.items() if k not in _EVIDENCE_SKIP and v is not None}
if extras:
kv = ", ".join(f"{k}={v}" for k, v in list(extras.items())[:6])
parts.append(kv)
return " | ".join(parts)
@@ -0,0 +1,14 @@
"""Machine-readable JSON report renderer."""
from __future__ import annotations
import json
from ..engine import AnalysisResult
def render(result: AnalysisResult, extra: dict | None = None) -> str:
data = result.to_dict()
if extra:
data["cloud_results"] = extra
return json.dumps(data, indent=2, ensure_ascii=False, default=str)
@@ -0,0 +1,111 @@
"""Markdown report renderer."""
from __future__ import annotations
from ..engine import AnalysisResult
from ..findings import Severity
from ._common import (
SEVERITY_LABEL, fmt_ts, affected_line, evidence_line, artifact_line, sources_list,
)
def render(result: AnalysisResult, extra: dict | None = None) -> str:
m = result.meta
s = result.summary
out: list[str] = []
out.append("# IoT PreTester 4.6 Demo — Network Capture Security Analysis")
out.append("")
out.append(f"- **Capture:** `{m.get('capture_path', '—')}`")
out.append(f"- **Packets:** {m.get('packet_count', 0)} · "
f"**Duration:** {m.get('duration_seconds', 0)} s · "
f"**Findings:** {m.get('finding_count', 0)}")
out.append("")
bysev = s.get("by_severity", {})
out.append("## Summary")
out.append("")
out.append("| Severity | Count |")
out.append("| ----------- | ------ |")
for sev in Severity:
out.append(f"| {SEVERITY_LABEL[sev]} | {bysev.get(sev.value, 0)} |")
out.append("")
highlights = s.get("highlights", [])
if highlights:
out.append("## ⚠️ Prioritized Highlights")
out.append("")
out.append("Highest-impact credential, token, C2, and exfiltration findings:")
out.append("")
for f in highlights:
out.append(f"- **[{SEVERITY_LABEL[f.severity]}/{f.confidence.value}]** "
f"{f.title} — {affected_line(f)}")
out.append("")
out.append("## Findings")
out.append("")
if not result.findings:
out.append("_No security-relevant findings detected._")
for i, f in enumerate(result.findings, 1):
out.append(f"### {i}. {f.title}")
out.append("")
out.append(f"- **Severity:** {SEVERITY_LABEL[f.severity]} "
f"· **Confidence:** {f.confidence.value} "
f"· **Category:** {f.category.value} "
f"· **Detector:** {f.detector}")
art = artifact_line(f)
if art and art != "—":
out.append(f"- **🔎 Observed artifact:** `{art}`")
srcs = sources_list(f)
if srcs:
out.append("- **📍 Sources:**")
for source in srcs:
out.append(f" - {source}")
out.append(f"- **Risk:** {f.risk_description}")
out.append(f"- **Affected:** {affected_line(f)}")
out.append(f"- **Additional evidence:** {evidence_line(f)}")
out.append(f"- **Recommendation:** {f.recommendation}")
out.append(f"- **Finding ID:** `{f.finding_id}`")
out.append("")
# Optional URL and hash enrichment results.
if extra:
hashes = extra.get("hashes") or []
urls = extra.get("urls") or []
if hashes:
out.append("")
out.append(f"## Hashes ({len(hashes)})")
out.append("")
for h in hashes:
if h.get("cracked"):
state = f"**CRACKED** → `{h.get('cracked_value')}`"
if h.get("crack_time"):
state += f" ({h['crack_time']})"
elif h.get("analyzed"):
state = "not cracked"
else:
state = "analysis pending"
htype = ", ".join(h.get("hash_type") or [])
out.append(f"- `{h.get('hash_value')}`"
f"{f' [{htype}]' if htype else ''} — {state}")
if urls:
out.append("")
out.append(f"## Cloud URLs and Endpoints ({len(urls)})")
out.append("")
for u in urls:
if u.get("analyzed"):
ports = ", ".join(map(str, u.get("ports") or [])) or "—"
details = f"Ports: {ports}"
if u.get("tls_versions"):
details += f"; TLS: {', '.join(u['tls_versions'])}"
if u.get("cipher_suites"):
details += f"; Cipher: {', '.join(u['cipher_suites'])}"
else:
details = "analysis pending"
out.append(f"- `{u.get('url')}` — {details}")
out.append("")
out.append("---")
out.append(f"_Generated by IoT PreTester 4.6 Demo · Detectors: "
f"{', '.join(m.get('detectors_run', []))}_")
return "\n".join(out)
+100
View File
@@ -0,0 +1,100 @@
"""Plain-text report renderer."""
from __future__ import annotations
from ..engine import AnalysisResult
from ..findings import Severity
from ._common import (
SEVERITY_LABEL, affected_line, evidence_line, artifact_line, sources_list,
)
def render(result: AnalysisResult, extra: dict | None = None) -> str:
m = result.meta
s = result.summary
out: list[str] = []
line = "=" * 70
out.append(line)
out.append("IoT PreTester 4.6 Demo - Network Capture Security Analysis")
out.append(line)
out.append(f"Capture : {m.get('capture_path', '-')}")
out.append(f"Packets : {m.get('packet_count', 0)} Duration: {m.get('duration_seconds', 0)}s")
out.append(f"Findings: {m.get('finding_count', 0)}")
out.append("")
bysev = s.get("by_severity", {})
out.append("SUMMARY")
out.append("-" * 70)
for sev in Severity:
out.append(f" {SEVERITY_LABEL[sev]:<8}: {bysev.get(sev.value, 0)}")
out.append("")
highlights = s.get("highlights", [])
if highlights:
out.append("PRIORITIZED HIGHLIGHTS")
out.append("-" * 70)
for f in highlights:
out.append(f" [{SEVERITY_LABEL[f.severity]}/{f.confidence.value}] "
f"{f.title} -- {affected_line(f)}")
out.append("")
out.append("FINDINGS")
out.append("-" * 70)
if not result.findings:
out.append(" No security-relevant findings detected.")
for i, f in enumerate(result.findings, 1):
out.append(f"[{i}] {f.title}")
out.append(f" Severity : {SEVERITY_LABEL[f.severity]} / Confidence: {f.confidence.value}")
out.append(f" Category : {f.category.value} (Detector: {f.detector})")
art = artifact_line(f)
if art and art != "—":
out.append(f" ARTIFACT : {art}")
srcs = sources_list(f)
if srcs:
out.append(" Sources :")
for source in srcs:
out.append(f" - {source}")
out.append(f" Risk : {f.risk_description}")
out.append(f" Affected : {affected_line(f)}")
out.append(f" Evidence : {evidence_line(f)}")
out.append(f" Recommendation : {f.recommendation}")
out.append(f" Finding ID : {f.finding_id}")
out.append("")
if extra:
hashes = extra.get("hashes") or []
urls = extra.get("urls") or []
if hashes:
out.append(f"HASHES ({len(hashes)})")
out.append("-" * 70)
for h in hashes:
if h.get("cracked"):
state = f"CRACKED -> {h.get('cracked_value')}"
if h.get("crack_time"):
state += f" ({h['crack_time']})"
elif h.get("analyzed"):
state = "not cracked"
else:
state = "analysis pending"
htype = ", ".join(h.get("hash_type") or [])
out.append(f" {h.get('hash_value')}{f' [{htype}]' if htype else ''}")
out.append(f" {state}")
out.append("")
if urls:
out.append(f"CLOUD URLS & ENDPOINTS ({len(urls)})")
out.append("-" * 70)
for u in urls:
out.append(f" {u.get('url')}")
if u.get("analyzed"):
ports = ", ".join(map(str, u.get("ports") or [])) or "-"
out.append(f" Ports: {ports}")
if u.get("tls_versions"):
out.append(f" TLS: {', '.join(u['tls_versions'])}")
if u.get("cipher_suites"):
out.append(f" Cipher: {', '.join(u['cipher_suites'])}")
else:
out.append(" analysis pending")
out.append("")
return "\n".join(out)
+80
View File
@@ -0,0 +1,80 @@
"""Central finding deduplication, prioritization, and summary logic."""
from __future__ import annotations
from .findings import Finding, Category, Severity, Confidence
# Categories highlighted in the report summary.
HIGHLIGHT_CATEGORIES = {
Category.PLAINTEXT_CREDENTIALS,
Category.PASSWORD_HASHES,
Category.TOKENS_SECRETS,
Category.MALWARE_BEACONING,
Category.EXFILTRATION,
}
def dedupe(findings: list[Finding]) -> list[Finding]:
"""Merge findings with the same ID while retaining their strongest evidence."""
by_id: dict[str, Finding] = {}
for f in findings:
existing = by_id.get(f.finding_id)
if existing is None:
by_id[f.finding_id] = f
continue
existing.timestamp_first_seen = _min(existing.timestamp_first_seen, f.timestamp_first_seen)
existing.timestamp_last_seen = _max(existing.timestamp_last_seen, f.timestamp_last_seen)
existing.related_packets = sorted(set(existing.related_packets) | set(f.related_packets))
existing.related_streams = sorted(set(existing.related_streams) | set(f.related_streams))
if f.confidence.rank > existing.confidence.rank:
existing.confidence = f.confidence
existing.evidence.setdefault("occurrences", 1)
existing.evidence["occurrences"] += 1
return list(by_id.values())
def sort_by_priority(findings: list[Finding]) -> list[Finding]:
"""Sort by severity, confidence, and stable category order."""
return sorted(
findings,
key=lambda f: (f.severity.rank, f.confidence.rank, -_category_order(f.category)),
reverse=True,
)
def summarize(findings: list[Finding]) -> dict:
"""Build the prioritized summary consumed by report renderers."""
by_sev = {s.value: 0 for s in Severity}
by_cat: dict[str, int] = {}
for f in findings:
by_sev[f.severity.value] += 1
by_cat[f.category.value] = by_cat.get(f.category.value, 0) + 1
highlights = [
f for f in sort_by_priority(findings)
if f.category in HIGHLIGHT_CATEGORIES
and f.severity.rank >= Severity.HIGH.rank
]
return {
"total": len(findings),
"by_severity": by_sev,
"by_category": by_cat,
"highlights": highlights,
"top": sort_by_priority(findings)[:10],
}
def _category_order(cat: Category) -> int:
order = list(Category)
return order.index(cat)
def _min(a, b):
vals = [x for x in (a, b) if x is not None]
return min(vals) if vals else None
def _max(a, b):
vals = [x for x in (a, b) if x is not None]
return max(vals) if vals else None
@@ -0,0 +1,879 @@
"""Unit tests for the framework-independent analysis core.
The tests use synthetic capture contexts and therefore require NO
tshark or pyshark. Run with:
cd django_project
python -m unittest analyzer.tests.test_analyzer -v
"""
import base64
import unittest
from analyzer import AnalysisEngine, render, FORMATS
from analyzer.context import (
CaptureContext, Conversation, DnsQuery, HttpRequest, HttpResponse,
TlsSession, ArpEvent, PlaintextAuth, AuthMaterial, Host,
)
from analyzer.findings import Finding, Category, Severity, Confidence
from analyzer import scoring
def run(ctx):
return AnalysisEngine().analyze_context(ctx)
def categories(result):
return {f.category for f in result.findings}
class PlaintextCredentialTests(unittest.TestCase):
def test_ftp_plaintext(self):
ctx = CaptureContext()
ctx.plaintext_auth.append(PlaintextAuth(
protocol="ftp", username="admin", password="secret",
src_ip="192.168.1.10", dst_ip="192.168.1.1", dst_port=21))
res = run(ctx)
self.assertIn(Category.PLAINTEXT_CREDENTIALS, categories(res))
f = next(f for f in res.findings if f.category == Category.PLAINTEXT_CREDENTIALS)
self.assertEqual(f.severity, Severity.CRITICAL)
self.assertEqual(f.evidence["username"], "admin")
self.assertNotEqual(f.evidence["password"], "secret") # Masked in evidence.
def test_http_basic_auth(self):
token = base64.b64encode(b"bob:hunter2").decode()
ctx = CaptureContext()
ctx.http_requests.append(HttpRequest(
method="GET", host="example.com", uri="/admin",
authorization=f"Basic {token}", src_ip="10.0.0.5", dst_ip="93.184.216.34",
dst_port=80))
res = run(ctx)
f = next(f for f in res.findings if "HTTP Basic" in f.title)
self.assertEqual(f.evidence["username"], "bob")
class TokenSecretTests(unittest.TestCase):
def test_jwt_detected(self):
jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.abcDEF123_-xyz"
ctx = CaptureContext()
ctx.text_payloads.append({"text": f"Authorization: Bearer {jwt}",
"protocol": "http", "src_ip": "10.0.0.1",
"dst_ip": "1.2.3.4", "packet_number": 7})
res = run(ctx)
self.assertIn(Category.TOKENS_SECRETS, categories(res))
def test_session_cookie_over_http(self):
ctx = CaptureContext()
ctx.text_payloads.append({"text": "Cookie: sessionid=ABCDEF1234567890",
"protocol": "http", "src_ip": "10.0.0.1",
"dst_ip": "1.2.3.4"})
res = run(ctx)
self.assertTrue(any("Session cookie" in f.title for f in res.findings))
class InsecureProtocolTests(unittest.TestCase):
def test_telnet_flagged_critical(self):
ctx = CaptureContext()
ctx.conversations.append(Conversation(
src_ip="192.168.0.2", dst_ip="192.168.0.3", src_port=50000, dst_port=23,
protocol="tcp", packets=10, bytes=500))
res = run(ctx)
f = next(f for f in res.findings if f.category == Category.INSECURE_PROTOCOL)
self.assertEqual(f.severity, Severity.CRITICAL)
self.assertEqual(f.port, 23)
class TlsWeaknessTests(unittest.TestCase):
def test_old_tls_version(self):
ctx = CaptureContext()
ctx.tls_sessions.append(TlsSession(
src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
version="TLS 1.0", cipher="TLS_RSA_WITH_RC4_128_SHA", sni="legacy.example"))
res = run(ctx)
titles = [f.title for f in res.findings if f.category == Category.TLS_WEAKNESS]
self.assertTrue(any("Obsolete TLS" in t for t in titles))
self.assertTrue(any("cipher" in t.lower() for t in titles))
class DnsAnomalyTests(unittest.TestCase):
def test_dns_tunneling_many_subdomains(self):
ctx = CaptureContext()
for i in range(40):
ctx.dns_queries.append(DnsQuery(
query_name=f"data{i}chunk.tunnel.example.com", rcode="NOERROR",
src_ip="10.0.0.9", dst_ip="8.8.8.8"))
res = run(ctx)
self.assertIn(Category.DNS_ANOMALY, categories(res))
def test_high_entropy_domain(self):
ctx = CaptureContext()
ctx.dns_queries.append(DnsQuery(
query_name="a8f3kd92jfh3kd0fj2k3.malware-c2.net", rcode="NOERROR"))
res = run(ctx)
self.assertIn(Category.DNS_ANOMALY, categories(res))
class MitmTests(unittest.TestCase):
def test_arp_spoof_multiple_macs(self):
ctx = CaptureContext()
ctx.arp_events.append(ArpEvent(sender_ip="192.168.1.1", sender_mac="aa:aa:aa:aa:aa:aa", opcode=2))
ctx.arp_events.append(ArpEvent(sender_ip="192.168.1.1", sender_mac="bb:bb:bb:bb:bb:bb", opcode=2))
res = run(ctx)
self.assertIn(Category.MITM, categories(res))
class BeaconingTests(unittest.TestCase):
def test_regular_beaconing(self):
ctx = CaptureContext()
conv = Conversation(src_ip="192.168.1.50", dst_ip="8.8.8.8", src_port=44000,
dst_port=443, protocol="tcp")
# Exact 60-second intervals produce a strong periodicity signal.
conv.timestamps = [float(t * 60) for t in range(8)]
conv.first_seen, conv.last_seen = conv.timestamps[0], conv.timestamps[-1]
ctx.conversations.append(conv)
res = run(ctx)
self.assertIn(Category.MALWARE_BEACONING, categories(res))
class ExfiltrationTests(unittest.TestCase):
def test_large_upload(self):
ctx = CaptureContext()
ctx.conversations.append(Conversation(
src_ip="192.168.1.20", dst_ip="8.8.8.8", src_port=40000, dst_port=443,
protocol="tcp", packets=100, bytes=10 * 1024 * 1024,
bytes_src_to_dst=10 * 1024 * 1024))
res = run(ctx)
self.assertIn(Category.EXFILTRATION, categories(res))
class BruteforceScanTests(unittest.TestCase):
def test_portscan(self):
ctx = CaptureContext()
for port in range(1000, 1030):
ctx.conversations.append(Conversation(
src_ip="10.0.0.66", dst_ip="10.0.0.5", src_port=50000 + port,
dst_port=port, protocol="tcp", packets=1, bytes=60))
res = run(ctx)
self.assertTrue(any("Port scan" in f.title for f in res.findings))
def test_malware_download(self):
ctx = CaptureContext()
ctx.http_requests.append(HttpRequest(
method="GET", host="evil.example", uri="/payload.exe",
src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=80))
res = run(ctx)
self.assertTrue(any(".exe" in f.title for f in res.findings))
class PasswordHashTests(unittest.TestCase):
def test_ntlm_material(self):
ctx = CaptureContext()
ctx.auth_material.append(AuthMaterial(
kind="ntlmssp", account="user1", domain="CORP",
src_ip="10.0.0.1", dst_ip="10.0.0.2", protocol="ntlm"))
res = run(ctx)
self.assertIn(Category.PASSWORD_HASHES, categories(res))
class ScoringTests(unittest.TestCase):
def _f(self, sev, conf=Confidence.HIGH, key="x"):
return Finding(title="t", category=Category.EXFILTRATION, severity=sev,
confidence=conf, risk_description="r", recommendation="rec",
evidence={"dedup_key": key})
def test_dedupe_merges_identical(self):
a = self._f(Severity.HIGH, Confidence.LOW, key="same")
b = self._f(Severity.HIGH, Confidence.HIGH, key="same")
a.related_packets = [1]
b.related_packets = [2]
merged = scoring.dedupe([a, b])
self.assertEqual(len(merged), 1)
self.assertEqual(merged[0].confidence, Confidence.HIGH)
self.assertEqual(merged[0].related_packets, [1, 2])
def test_sort_by_priority(self):
low = self._f(Severity.LOW, key="a")
crit = self._f(Severity.CRITICAL, key="b")
med = self._f(Severity.MEDIUM, key="c")
ordered = scoring.sort_by_priority([low, crit, med])
self.assertEqual([f.severity for f in ordered],
[Severity.CRITICAL, Severity.MEDIUM, Severity.LOW])
class ReportRenderingTests(unittest.TestCase):
def _result(self):
ctx = CaptureContext(path="x.pcapng", packet_count=10)
ctx.plaintext_auth.append(PlaintextAuth(
protocol="ftp", username="admin", password="secret",
src_ip="192.168.1.10", dst_ip="192.168.1.1", dst_port=21))
return run(ctx)
def test_all_formats_render(self):
res = self._result()
for fmt in FORMATS:
out = render(res, fmt)
self.assertIsInstance(out, str)
self.assertTrue(len(out) > 0)
def test_json_is_valid(self):
import json
res = self._result()
data = json.loads(render(res, "json"))
self.assertIn("findings", data)
self.assertIn("summary", data)
self.assertGreaterEqual(len(data["findings"]), 1)
def test_extra_cloud_results_in_report(self):
import json
res = self._result()
extra = {
"urls": [{"url": "api.example.com", "analyzed": True, "ports": [443],
"tls_versions": ["TLS 1.2"], "cipher_suites": ["AES_256_GCM"]}],
"hashes": [{"hash_value": "5f4dcc3b5aa765d61d8327deb882cf99", "analyzed": True,
"cracked": True, "cracked_value": "password", "crack_time": "0.10s",
"hash_type": ["md5"]}],
}
md = render(res, "md", extra)
self.assertIn("api.example.com", md)
self.assertIn("password", md)
txt = render(res, "txt", extra)
self.assertIn("CRACKED", txt)
data = json.loads(render(res, "json", extra))
self.assertIn("cloud_results", data)
self.assertEqual(data["cloud_results"]["hashes"][0]["cracked_value"], "password")
class ReconstructionTests(unittest.TestCase):
def test_roundtrip_from_dict_renders(self):
import json
from analyzer import result_from_dict, render
ctx = CaptureContext(path="x.pcapng", packet_count=5)
ctx.plaintext_auth.append(PlaintextAuth(
protocol="ftp", username="admin", password="secret",
src_ip="192.168.1.10", dst_ip="192.168.1.1", dst_port=21))
res = run(ctx)
# Serialize and reconstruct a stored result.
stored = json.dumps(res.to_dict())
rebuilt = result_from_dict(json.loads(stored))
self.assertEqual(len(rebuilt.findings), len(res.findings))
self.assertEqual(rebuilt.findings[0].finding_id, res.findings[0].finding_id)
# Every renderer must work with the reconstructed result.
for fmt in FORMATS:
out = render(rebuilt, fmt)
self.assertTrue(len(out) > 0)
class EngineRobustnessTests(unittest.TestCase):
def test_empty_capture(self):
res = run(CaptureContext(path="empty.pcap"))
self.assertEqual(res.findings, [])
self.assertEqual(res.summary["total"], 0)
def test_all_ten_detectors_registered(self):
res = run(CaptureContext())
self.assertEqual(len(res.meta["detectors_run"]), 10)
self.assertEqual(res.meta["detector_errors"], {})
# --------------------------------------------------------------------------- #
# Detector regression tests
# --------------------------------------------------------------------------- #
class BeaconingConnectionTests(unittest.TestCase):
def test_beaconing_via_distinct_connections(self):
"""Detect TCP beaconing spread across multiple short connections."""
ctx = CaptureContext()
for i in range(8):
t = float(i * 30) # One new connection every 30 seconds.
ctx.conversations.append(Conversation(
src_ip="192.168.1.77", dst_ip="8.8.8.8",
src_port=40000 + i, dst_port=443, protocol="tcp",
packets=2, bytes=200, first_seen=t, last_seen=t + 1,
stream_id=i, timestamps=[t, t + 1],
))
res = run(ctx)
beacons = [f for f in res.findings if f.category == Category.MALWARE_BEACONING]
self.assertTrue(beacons, "Beaconing across separate connections was not detected")
self.assertEqual(beacons[0].evidence.get("signal"), "distinct_connections")
self.assertEqual(beacons[0].destination_ip, "8.8.8.8")
def test_irregular_connections_not_flagged(self):
ctx = CaptureContext()
for i, t in enumerate([0.0, 5.0, 90.0, 91.0, 400.0, 402.0]):
ctx.conversations.append(Conversation(
src_ip="192.168.1.77", dst_ip="8.8.8.8",
src_port=40000 + i, dst_port=443, protocol="tcp",
first_seen=t, last_seen=t, timestamps=[t]))
res = run(ctx)
self.assertNotIn(Category.MALWARE_BEACONING, categories(res))
class ProtocolStatsCaseTests(unittest.TestCase):
def test_eapol_handshake_real_casing(self):
"""Protocol statistics remain case-insensitive for tshark layer names."""
ctx = CaptureContext()
ctx.protocol_stats["EAPOL"] = 4
res = run(ctx)
self.assertTrue(any("EAPOL" in f.title for f in res.findings))
def test_smbv1_real_casing(self):
ctx = CaptureContext()
ctx.protocol_stats["SMB"] = 20
res = run(ctx)
self.assertTrue(any("SMBv1" in f.title for f in res.findings))
def test_smbv1_suppressed_when_smb2_present(self):
ctx = CaptureContext()
ctx.protocol_stats["SMB"] = 20
ctx.protocol_stats["SMB2"] = 100
res = run(ctx)
self.assertFalse(any("SMBv1" in f.title for f in res.findings))
class TlsCertTests(unittest.TestCase):
def test_self_signed_certificate_flagged(self):
ctx = CaptureContext()
ctx.tls_sessions.append(TlsSession(
src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
version="TLS 1.2", sni="iot.local",
cert_subject_cn="iot.local", cert_issuer_cn="iot.local",
cert_self_signed=True))
res = run(ctx)
self.assertTrue(any("self-signed" in f.title for f in res.findings))
def test_expired_certificate_flagged(self):
ctx = CaptureContext()
ctx.tls_sessions.append(TlsSession(
src_ip="10.0.0.1", dst_ip="1.2.3.4", dst_port=443,
version="TLS 1.2", cert_expired=True))
res = run(ctx)
self.assertTrue(any("expired" in f.title for f in res.findings))
class CertHelperTests(unittest.TestCase):
def test_self_signed_helper(self):
from analyzer.extract_pyshark import _cert_is_self_signed
self.assertTrue(_cert_is_self_signed("device.local", "device.local"))
self.assertFalse(_cert_is_self_signed("device.local", "DigiCert CA"))
self.assertIsNone(_cert_is_self_signed("device.local", None))
def test_expired_helper(self):
from analyzer.extract_pyshark import _cert_is_expired
# The certificate expired before the 2025 capture timestamp.
now = 1735689600.0 # 2025-01-01 UTC
self.assertTrue(_cert_is_expired("2020-01-01 00:00:00", now))
self.assertFalse(_cert_is_expired("2030-01-01 00:00:00", now))
self.assertIsNone(_cert_is_expired(None, now))
def test_parse_cert_time_formats(self):
from analyzer.extract_pyshark import _parse_cert_time
self.assertIsNotNone(_parse_cert_time("2020-01-01 00:00:00"))
self.assertIsNotNone(_parse_cert_time("Jan 1 00:00:00 2020 GMT"))
self.assertIsNotNone(_parse_cert_time("200101000000Z"))
self.assertIsNone(_parse_cert_time("nonsense"))
# --------------------------------------------------------------------------- #
# Authentication material reconstruction
# --------------------------------------------------------------------------- #
class CrackFormatTests(unittest.TestCase):
def test_netntlmv2(self):
from analyzer.crackformats import build_netntlmv2
nt = "a" * 32 + "b" * 40 # NTProof(16B) + Blob
out = build_netntlmv2("alice", "CORP", "1122334455667788", nt)
self.assertIsNotNone(out)
self.assertEqual(out["hashcat_mode"], 5600)
self.assertEqual(out["hashcat"],
"alice::CORP:1122334455667788:" + "a" * 32 + ":" + "b" * 40)
def test_netntlmv2_rejects_bad_challenge(self):
from analyzer.crackformats import build_netntlmv2
self.assertIsNone(build_netntlmv2("alice", "CORP", "1122", "a" * 40))
def test_krb5_asrep(self):
from analyzer.crackformats import build_krb5_asrep
cipher = "c" * 32 + "d" * 64
out = build_krb5_asrep("bob", "corp.local", 23, cipher)
self.assertEqual(out["hashcat_mode"], 18200)
self.assertTrue(out["hashcat"].startswith("$krb5asrep$23$bob@CORP.LOCAL:"))
def test_krb5_tgsrep(self):
from analyzer.crackformats import build_krb5_tgsrep
cipher = "e" * 32 + "f" * 64
out = build_krb5_tgsrep("svc", "corp.local", "MSSQL/db", 23, cipher)
self.assertEqual(out["hashcat_mode"], 13100)
self.assertIn("$krb5tgs$23$*svc$CORP.LOCAL$MSSQL/db*$", out["hashcat"])
def test_smtp_auth_plain(self):
import base64
from analyzer.crackformats import decode_smtp_auth_plain
token = base64.b64encode(b"\x00joe\x00s3cret").decode()
self.assertEqual(decode_smtp_auth_plain(token), ("joe", "s3cret"))
def test_smtp_auth_login(self):
import base64
from analyzer.crackformats import decode_smtp_auth_login
u = base64.b64encode(b"joe").decode()
p = base64.b64encode(b"s3cret").decode()
self.assertEqual(decode_smtp_auth_login(u, p), ("joe", "s3cret"))
class CrackableAuthMaterialTests(unittest.TestCase):
def test_hashcat_material_surfaced_and_high_confidence(self):
ctx = CaptureContext()
ctx.auth_material.append(AuthMaterial(
kind="netntlmv2", account="alice", domain="CORP",
src_ip="10.0.0.5", dst_ip="10.0.0.1", protocol="ntlm",
hashcat="alice::CORP:1122334455667788:" + "a" * 32 + ":" + "b" * 40,
hashcat_mode=5600, hash_format="NetNTLMv2"))
res = run(ctx)
ph = [f for f in res.findings if f.category == Category.PASSWORD_HASHES]
self.assertTrue(ph)
f = ph[0]
self.assertEqual(f.confidence, Confidence.HIGH)
self.assertIn("crackable", f.title)
self.assertEqual(f.evidence.get("hashcat_mode"), 5600)
self.assertTrue(f.evidence.get("hashcat", "").startswith("alice::CORP:"))
class MailPlaintextAuthTests(unittest.TestCase):
def test_pop3_plaintext(self):
ctx = CaptureContext()
ctx.plaintext_auth.append(PlaintextAuth(
protocol="pop3", username="joe", password="pw123",
src_ip="10.0.0.5", dst_ip="10.0.0.2", dst_port=110))
res = run(ctx)
self.assertTrue(any("POP3" in f.title for f in res.findings))
def test_smtp_plaintext(self):
ctx = CaptureContext()
ctx.plaintext_auth.append(PlaintextAuth(
protocol="smtp", username="joe", password="pw123",
src_ip="10.0.0.5", dst_ip="10.0.0.2", dst_port=25))
res = run(ctx)
self.assertTrue(any("SMTP" in f.title for f in res.findings))
# --------------------------------------------------------------------------- #
# False-positive regression tests
# --------------------------------------------------------------------------- #
class UserAgentTests(unittest.TestCase):
def _ua(self, ua):
ctx = CaptureContext()
ctx.http_requests.append(HttpRequest(
method="GET", host="x.example", uri="/", user_agent=ua,
src_ip="10.0.0.5", dst_ip="1.2.3.4", dst_port=80))
return run(ctx)
def test_benign_iot_ua_not_flagged(self):
for ua in ("okhttp/4.9.0", "Java/1.8.0", "Go-http-client/1.1"):
res = self._ua(ua)
self.assertFalse(
any(f.category == Category.MALWARE_BEACONING for f in res.findings),
f"{ua} sollte kein Finding erzeugen")
def test_scanner_ua_flagged_high(self):
res = self._ua("Mozilla/5.00 (Nikto/2.1.6)")
ua = [f for f in res.findings if f.category == Category.MALWARE_BEACONING]
self.assertTrue(ua)
self.assertEqual(ua[0].severity, Severity.HIGH)
self.assertEqual(ua[0].evidence.get("tier"), "scanner")
def test_scripting_ua_flagged_low(self):
res = self._ua("curl/8.1.2")
ua = [f for f in res.findings if f.category == Category.MALWARE_BEACONING]
self.assertTrue(ua)
self.assertEqual(ua[0].severity, Severity.LOW)
class DnsQtypeTests(unittest.TestCase):
def test_qtype_name_mapping(self):
from analyzer.extract_pyshark import _dns_qtype_name
self.assertEqual(_dns_qtype_name("16"), "TXT")
self.assertEqual(_dns_qtype_name("1"), "A")
self.assertEqual(_dns_qtype_name("txt"), "TXT")
self.assertIsNone(_dns_qtype_name(None))
def test_txt_tunneling_flagged(self):
ctx = CaptureContext()
for i in range(25):
ctx.dns_queries.append(DnsQuery(
query_name=f"chunk{i}.exfil.example.com", qtype="TXT",
rcode="NOERROR", src_ip="10.0.0.9", dst_ip="8.8.8.8"))
res = run(ctx)
self.assertTrue(any("TXT or NULL" in f.title for f in res.findings))
def test_normal_a_queries_no_txt_finding(self):
ctx = CaptureContext()
for i in range(25):
ctx.dns_queries.append(DnsQuery(
query_name=f"host{i}.example.com", qtype="A", rcode="NOERROR"))
res = run(ctx)
self.assertFalse(any("TXT or NULL" in f.title for f in res.findings))
class DownloadHelperTests(unittest.TestCase):
def test_is_download_content_type(self):
from analyzer.extract_pyshark import _is_download_content_type
self.assertTrue(_is_download_content_type("application/octet-stream"))
self.assertTrue(_is_download_content_type("application/x-msdownload"))
self.assertFalse(_is_download_content_type("text/html; charset=utf-8"))
self.assertFalse(_is_download_content_type(None))
def test_filename_from_disposition(self):
from analyzer.extract_pyshark import _filename_from_disposition
self.assertEqual(_filename_from_disposition('attachment; filename="update.exe"'), "update.exe")
self.assertEqual(_filename_from_disposition("attachment; filename=payload.bin"), "payload.bin")
self.assertIsNone(_filename_from_disposition("inline"))
class FileTransferDownloadTests(unittest.TestCase):
def test_executable_content_type_flagged(self):
from analyzer.context import FileTransfer
ctx = CaptureContext()
ctx.file_transfers.append(FileTransfer(
filename=None, extension=None, content_type="application/x-msdownload",
protocol="http", src_ip="1.2.3.4", dst_ip="10.0.0.5", direction="download"))
res = run(ctx)
self.assertTrue(any("harmful file" in f.title for f in res.findings))
def test_disposition_filename_extension_flagged(self):
from analyzer.context import FileTransfer
ctx = CaptureContext()
ctx.file_transfers.append(FileTransfer(
filename="update.exe", extension=".exe", content_type="application/octet-stream",
protocol="http", src_ip="1.2.3.4", dst_ip="10.0.0.5", direction="download"))
res = run(ctx)
self.assertTrue(any(".exe" in f.title for f in res.findings))
def test_benign_download_not_flagged(self):
from analyzer.context import FileTransfer
ctx = CaptureContext()
ctx.file_transfers.append(FileTransfer(
filename="report.pdf", extension=".pdf", content_type="application/pdf",
protocol="http", src_ip="1.2.3.4", dst_ip="10.0.0.5", direction="download"))
res = run(ctx)
self.assertFalse(any(f.category == Category.BRUTEFORCE_SCAN_FILETRANSFER
for f in res.findings))
# --------------------------------------------------------------------------- #
# Packet-level artifact provenance
# --------------------------------------------------------------------------- #
class ProvenanceHelperTests(unittest.TestCase):
def test_derive_artifact_credentials(self):
from analyzer.provenance import derive_artifact
art = derive_artifact({"username": "joe", "password": "pw"})
self.assertEqual(art["label"], "Credentials")
self.assertIn("joe", art["value"])
def test_derive_artifact_hashcat(self):
from analyzer.provenance import derive_artifact
art = derive_artifact({"hashcat": "alice::CORP:...:...:..."})
self.assertEqual(art["label"], "Hash (hashcat)")
def test_derive_artifact_none_for_behavioral(self):
from analyzer.provenance import derive_artifact
self.assertIsNone(derive_artifact({"distinct_ports": 30}))
def test_build_sources_single_packet_has_timestamp(self):
from analyzer.provenance import build_sources
srcs = build_sources(related_packets=[42], related_streams=[3],
src_ip="10.0.0.5", dst_ip="10.0.0.2", protocol="ftp",
port=21, timestamp_first_seen=1577880000.0, capture="c.pcapng")
self.assertEqual(len(srcs), 1)
self.assertEqual(srcs[0]["packet"], 42)
self.assertEqual(srcs[0]["stream"], 3)
self.assertEqual(srcs[0]["timestamp"], 1577880000.0)
self.assertEqual(srcs[0]["capture"], "c.pcapng")
def test_build_sources_multi_packet_capped_no_per_packet_ts(self):
from analyzer.provenance import build_sources, MAX_SOURCE_PACKETS
srcs = build_sources(related_packets=list(range(1, 20)), related_streams=[1, 2],
src_ip="10.0.0.5", dst_ip="8.8.8.8", protocol="tcp",
port=443, timestamp_first_seen=100.0, capture="c.pcapng")
self.assertEqual(len(srcs), MAX_SOURCE_PACKETS)
self.assertNotIn("timestamp", srcs[0]) # Never infer per-packet timestamps.
self.assertNotIn("stream", srcs[0]) # Omit an ambiguous stream reference.
def test_build_sources_no_packet_still_has_reference(self):
from analyzer.provenance import build_sources
srcs = build_sources(related_packets=[], related_streams=[],
src_ip="10.0.0.9", dst_ip=None, protocol="dns", port=53,
timestamp_first_seen=None, capture="c.pcapng")
self.assertEqual(len(srcs), 1)
self.assertIn("note", srcs[0])
class EngineProvenanceTests(unittest.TestCase):
def test_finding_gets_capture_artifact_sources(self):
ctx = CaptureContext(path="/data/demo_capture.pcapng")
ctx.plaintext_auth.append(PlaintextAuth(
protocol="ftp", username="admin", password="s3cret",
src_ip="10.0.0.5", dst_ip="10.0.0.2", dst_port=21,
timestamp=1577880000.0, packet_number=42, stream_id=3))
res = run(ctx)
f = next(f for f in res.findings if f.category == Category.PLAINTEXT_CREDENTIALS)
# Store only the capture basename.
self.assertEqual(f.evidence["capture_file"], "demo_capture.pcapng")
self.assertEqual(f.evidence["artifact"]["label"], "Credentials")
self.assertTrue(f.evidence["sources"])
self.assertEqual(f.evidence["sources"][0]["packet"], 42)
self.assertEqual(f.evidence["sources"][0]["capture"], "demo_capture.pcapng")
def test_aggregated_findings_have_packets(self):
# Aggregated scan findings retain representative packet numbers.
ctx = CaptureContext(path="cap.pcapng")
for p in range(1000, 1020):
ctx.conversations.append(Conversation(
src_ip="10.0.0.66", dst_ip="10.0.0.5", src_port=50000 + p,
dst_port=p, protocol="tcp", packets=1, bytes=60,
packet_numbers=[p]))
res = run(ctx)
scan = next(f for f in res.findings if "Port scan" in f.title)
self.assertTrue(scan.related_packets, "Port scan has no packet evidence")
self.assertTrue(scan.evidence["sources"])
self.assertIn("packet", scan.evidence["sources"][0])
def test_beaconing_distinct_connections_have_packets(self):
ctx = CaptureContext(path="cap.pcapng")
for i in range(8):
t = float(i * 30)
ctx.conversations.append(Conversation(
src_ip="192.168.1.77", dst_ip="8.8.8.8", src_port=40000 + i,
dst_port=443, protocol="tcp", first_seen=t, last_seen=t,
timestamps=[t], stream_id=i, packet_numbers=[500 + i]))
res = run(ctx)
beacon = next(f for f in res.findings if f.category == Category.MALWARE_BEACONING)
self.assertTrue(beacon.related_packets)
class ReportProvenanceTests(unittest.TestCase):
def test_report_shows_artifact_and_packet(self):
ctx = CaptureContext(path="/data/evidence.pcapng")
ctx.plaintext_auth.append(PlaintextAuth(
protocol="ftp", username="admin", password="s3cret",
src_ip="10.0.0.5", dst_ip="10.0.0.2", dst_port=21,
timestamp=1577880000.0, packet_number=42, stream_id=3))
res = run(ctx)
md = render(res, "md")
self.assertIn("Observed artifact", md)
self.assertIn("Packet #42", md)
self.assertIn("evidence.pcapng", md)
txt = render(res, "txt")
self.assertIn("ARTIFACT", txt)
self.assertIn("Packet #42", txt)
# --------------------------------------------------------------------------- #
# Artifact extraction and enrichment integration
# --------------------------------------------------------------------------- #
class HashModeTests(unittest.TestCase):
def test_length_based_modes(self):
from analyzer.artifacts import identify_hash_mode
self.assertEqual(identify_hash_mode("a" * 32)[0], 0) # md5
self.assertEqual(identify_hash_mode("a" * 40)[0], 100) # sha1
self.assertEqual(identify_hash_mode("a" * 64)[0], 1400) # sha256
self.assertEqual(identify_hash_mode("a" * 128)[0], 1700) # sha512
def test_structural_modes(self):
from analyzer.artifacts import identify_hash_mode
netntlmv2 = "alice::CORP:" + "1" * 16 + ":" + "a" * 32 + ":" + "b" * 40
self.assertEqual(identify_hash_mode(netntlmv2)[0], 5600)
self.assertEqual(identify_hash_mode("$krb5asrep$23$bob@CORP:aaa$bbb")[0], 18200)
self.assertEqual(identify_hash_mode("$krb5tgs$23$*svc$CORP$spn*$a$b")[0], 13100)
self.assertEqual(identify_hash_mode("wpa*01*...")[0], 22000)
def test_hint_format_wins(self):
from analyzer.artifacts import identify_hash_mode
self.assertEqual(identify_hash_mode("whatever", hint_format="NetNTLMv2")[0], 5600)
self.assertEqual(identify_hash_mode("x", hint_format="krb5asrep(etype=23)")[0], 18200)
def test_unknown_returns_none(self):
from analyzer.artifacts import identify_hash_mode
self.assertEqual(identify_hash_mode("nothash"), (None, None))
self.assertEqual(identify_hash_mode("a" * 33), (None, None))
class ArtifactExtractionTests(unittest.TestCase):
def test_extract_urls_dedupes_and_observes(self):
from analyzer import extract_urls
ctx = CaptureContext()
ctx.dns_queries.append(DnsQuery(query_name="api.example.com.", src_ip="10.0.0.5"))
ctx.tls_sessions.append(TlsSession(src_ip="10.0.0.5", dst_ip="1.2.3.4",
dst_port=443, version="TLS 1.2", sni="api.example.com"))
ctx.http_requests.append(HttpRequest(method="GET", host="cdn.example.net",
uri="/x", src_ip="10.0.0.5", dst_ip="1.2.3.4", dst_port=80))
urls = extract_urls(ctx)
vals = {u.value for u in urls}
self.assertIn("api.example.com", vals)
self.assertIn("cdn.example.net", vals)
# DNS and SNI observations merge into one endpoint artifact.
self.assertEqual(len([u for u in urls if u.value == "api.example.com"]), 1)
api = next(u for u in urls if u.value == "api.example.com")
self.assertIn(443, api.observed_ports)
self.assertIn("TLS 1.2", api.observed_tls)
def test_extract_hashes_from_auth_material_and_payload(self):
from analyzer import extract_hashes
ctx = CaptureContext()
ntlm = "alice::CORP:" + "1" * 16 + ":" + "a" * 32 + ":" + "b" * 40
ctx.auth_material.append(AuthMaterial(
kind="netntlmv2", account="alice", hashcat=ntlm, hashcat_mode=5600,
hash_format="NetNTLMv2", src_ip="10.0.0.5", packet_number=7))
ctx.text_payloads.append({"text": "token=" + "d" * 64, "protocol": "http",
"src_ip": "10.0.0.5", "packet_number": 9})
hashes = extract_hashes(ctx)
modes = {h.hashcat_mode for h in hashes}
self.assertIn(5600, modes) # NTLM
self.assertIn(1400, modes) # SHA-256 extracted from payload text.
ntlm_art = next(h for h in hashes if h.hashcat_mode == 5600)
self.assertEqual(ntlm_art.account, "alice")
self.assertEqual(ntlm_art.kind, "netntlmv2")
class DemoPayloadRegressionTests(unittest.TestCase):
"""Extract hashes and URLs from generic HTTP body payloads."""
def test_hashes_and_urls_from_body_payload(self):
from analyzer import extract_urls, extract_hashes
ctx = CaptureContext()
ctx.text_payloads.append({"text": (
"POST /hashes HTTP/1.1\r\nHost: testserver\r\n\r\n"
"5f4dcc3b5aa765d61d8327deb882cf99\n" # md5
"e27f400cc2fe611bbb129ae0b4b4be6b32debf12\n" # sha1
"0e2cfcd348361a44a6f3e646552cb56d8d4d9555b4bcd89770f96983d8f321ad" # sha256
), "protocol": "http", "packet_number": 1})
ctx.text_payloads.append({"text": (
"GET /allowed_urls HTTP/1.1\r\nHost: testserver\r\n\r\n"
"eu.iot.dreame.tech\n"
"https://app.mt.eu.iot.dreame.tech\n"
"http://40.101.70.178/\n"
), "protocol": "http", "packet_number": 2})
hashes = extract_hashes(ctx)
self.assertEqual(len(hashes), 3)
self.assertEqual({h.hashcat_mode for h in hashes}, {0, 100, 1400})
urls = {u.value for u in extract_urls(ctx)}
self.assertIn("eu.iot.dreame.tech", urls)
self.assertIn("https://app.mt.eu.iot.dreame.tech", urls)
self.assertTrue(any("40.101.70.178" in u for u in urls))
def test_hexish_payload_decoder(self):
from analyzer.extract_pyshark import _bytes_from_hexish
# pyshark commonly exposes payload bytes as colon-separated hex.
self.assertEqual(_bytes_from_hexish("47:45:54"), "GET")
self.assertEqual(_bytes_from_hexish("already text"), "already text")
class HashCrackTests(unittest.TestCase):
def _runner_factory(self, show_stdout):
import types
calls = {"n": 0}
def runner(cmd, **kw):
calls["n"] += 1
# The second invocation reads the cracked result.
out = show_stdout if "--show" in cmd else ""
return types.SimpleNamespace(stdout=out, stderr="", returncode=0)
return runner, calls
def test_command_builders_and_parser(self):
from analyzer.enrich.hash_crack import build_crack_command, build_show_command, parse_show_output
cmd = build_crack_command("hashcat", 5600, "/tmp/h.txt", ["/wl/rockyou.txt"], "/tmp/pot")
self.assertEqual(cmd[:3], ["hashcat", "-m", "5600"])
self.assertIn("--potfile-path", cmd)
self.assertIn("--show", build_show_command("hashcat", 0, "/tmp/h.txt", "/tmp/pot"))
self.assertEqual(parse_show_output("abc123:hunter2"), "hunter2")
self.assertIsNone(parse_show_output(""))
def test_crack_success(self):
from analyzer.artifacts import HashArtifact
from analyzer.enrich.hash_crack import DEFAULT_WORDLISTS, crack_hash
runner, _ = self._runner_factory(show_stdout="deadbeef" * 8 + ":hunter2\n")
art = HashArtifact(value="d" * 64, hashcat_mode=1400, hash_format="sha256")
res = crack_hash(art, wordlists=DEFAULT_WORDLISTS, hashcat_path="hashcat",
runner=runner, which=lambda p: "/usr/bin/hashcat")
self.assertTrue(res.cracked)
self.assertEqual(res.plaintext, "hunter2")
self.assertEqual(res.state, "cracked")
def test_crack_not_found(self):
from analyzer.artifacts import HashArtifact
from analyzer.enrich.hash_crack import DEFAULT_WORDLISTS, crack_hash
runner, _ = self._runner_factory(show_stdout="")
art = HashArtifact(value="d" * 64, hashcat_mode=1400)
res = crack_hash(art, wordlists=DEFAULT_WORDLISTS, runner=runner,
which=lambda p: "/usr/bin/hashcat")
self.assertFalse(res.cracked)
self.assertEqual(res.state, "not_cracked")
def test_crack_unsupported_mode(self):
from analyzer.enrich.hash_crack import crack_hash
res = crack_hash("not-a-hash", which=lambda p: "/usr/bin/hashcat")
self.assertEqual(res.state, "unsupported")
def test_crack_hashcat_missing(self):
from analyzer.artifacts import HashArtifact
from analyzer.enrich.hash_crack import crack_hash
art = HashArtifact(value="d" * 64, hashcat_mode=1400)
res = crack_hash(art, wordlists=["/etc/hostname"], which=lambda p: None)
self.assertEqual(res.state, "error")
class UrlProbeTests(unittest.TestCase):
def test_host_of(self):
from analyzer.enrich.url_probe import host_of
self.assertEqual(host_of("https://api.example.com/x?y=1"), "api.example.com")
self.assertEqual(host_of("example.net:8080/path"), "example.net")
self.assertEqual(host_of("plain.host"), "plain.host")
def test_passive_returns_observed(self):
from analyzer.artifacts import UrlArtifact
from analyzer.enrich.url_probe import probe_url
art = UrlArtifact(value="api.example.com", kind="sni",
observed_ports={443}, observed_tls={"TLS 1.2"})
probe = probe_url(art, active=False)
self.assertFalse(probe.active)
self.assertEqual(probe.open_ports, [443])
self.assertEqual(probe.tls_versions, ["TLS 1.2"])
def test_active_merges_injected_scanners(self):
from analyzer.artifacts import UrlArtifact
from analyzer.enrich.url_probe import probe_url
art = UrlArtifact(value="api.example.com", observed_ports={443})
probe = probe_url(
art, active=True,
dns_resolver=lambda h: ["93.184.216.34"],
port_scanner=lambda h, ports: [80, 443],
tls_scanner=lambda h: {"tls_version": "TLS 1.3", "cipher": "TLS_AES_128_GCM_SHA256",
"certificate": "Present"},
)
self.assertTrue(probe.active)
self.assertEqual(probe.resolved_ips, ["93.184.216.34"])
self.assertEqual(probe.open_ports, [80, 443])
self.assertIn("TLS 1.3", probe.tls_versions)
self.assertEqual(probe.certificate, "Present")
class EnrichmentEngineTests(unittest.TestCase):
def test_engine_progress_and_results(self):
from analyzer.artifacts import HashArtifact
from analyzer.enrich import EnrichmentEngine
seen = []
arts = [HashArtifact(value="d" * 64, hashcat_mode=1400),
HashArtifact(value="nope", hashcat_mode=None)]
res = EnrichmentEngine().crack_hashes(
arts, progress=lambda i, t, item: seen.append((i, t)),
which=lambda p: None) # Avoid a real subprocess in this unit test.
self.assertEqual(len(res), 2)
self.assertEqual(seen, [(1, 2), (2, 2)])
self.assertEqual(res[1].state, "unsupported")
if __name__ == "__main__":
unittest.main(verbosity=2)
+65
View File
@@ -0,0 +1,65 @@
"""Seed the IoT PreTester 4.6 Demo reference data."""
import os
import django
from django.db import transaction
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "IoTPreTester.settings")
django.setup()
from IoTAnalyzer.models import Device, DeviceType, Manufacturer, Product # noqa: E402
DEVICE_TYPES = (
"Camera",
"Smart Plug",
"Smart Bulb",
"Smart Switch",
"Robot Vacuum Cleaner",
"Smart Lock",
"Smart Controller",
"Sensor",
)
MANUFACTURERS = (
"Aqara",
"Dreame",
"DJI",
"IKEA",
"Nuki",
"Philips Hue",
"Roborock",
"Shelly",
"Sonoff",
"TP-Link",
"X-Sense",
)
@transaction.atomic
def populate_database() -> None:
"""Create stable reference rows and one sample device without duplicates."""
for name in DEVICE_TYPES:
DeviceType.objects.get_or_create(device_type=name)
for name in MANUFACTURERS:
Manufacturer.objects.get_or_create(name=name)
manufacturer = Manufacturer.objects.get(name="Shelly")
device_type = DeviceType.objects.get(device_type="Smart Controller")
product, _ = Product.objects.get_or_create(
manufacturer=manufacturer,
device_type=device_type,
product_name="1PM Gen3",
)
Device.objects.get_or_create(
product=product,
serial="DEMO-001",
defaults={"comment": "IoT PreTester 4.6 Demo sample device"},
)
print("Demo reference data synchronized")
if __name__ == "__main__":
populate_database()
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "IoTPreTester.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == "__main__":
main()