v4.5 Demo initial commit
This commit is contained in:
DaStra3003
@@ -0,0 +1,626 @@
|
||||
import subprocess
|
||||
import os
|
||||
import re
|
||||
import math
|
||||
import logging
|
||||
from IoTAnalyzer.models import BinwalkRaw, StringsRaw, RawDataset, Tool
|
||||
from IoTAnalyzer.db_create import (
|
||||
create_binwalk_raw,
|
||||
create_strings_raw,
|
||||
create_analyzed_dataset,
|
||||
create_binwalk_analyzed,
|
||||
create_strings_analyzed,
|
||||
)
|
||||
import json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def analyze_firmware_file(firmware_file, raw_dataset_id):
|
||||
"""
|
||||
Main function for firmware analysis using binwalk and strings.
|
||||
"""
|
||||
print(f"Start analyzing firmware: {firmware_file}")
|
||||
logger.info(f"Firmware-Analyse gestartet: {firmware_file} (RawDataset ID: {raw_dataset_id})")
|
||||
|
||||
# Robust path resolution for the firmware file (multiple candidates)
|
||||
# repo_root = project root (3 levels up from analyze_firmware.py)
|
||||
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
|
||||
candidates = [
|
||||
# preferred: central files directory in the repo root
|
||||
os.path.join(repo_root, 'files', firmware_file),
|
||||
# Fallback: frontend public folder (Next.js)
|
||||
os.path.join(repo_root, 'Frontend', 'iot-pre-tester-frontend', 'public', 'filesAttachment', firmware_file),
|
||||
# Relative (in case the working directory is already the repo root)
|
||||
os.path.join('files', firmware_file),
|
||||
os.path.join('Frontend', 'iot-pre-tester-frontend', 'public', 'filesAttachment', firmware_file),
|
||||
]
|
||||
|
||||
filepath = None
|
||||
for cand in candidates:
|
||||
if os.path.exists(cand):
|
||||
filepath = cand
|
||||
break
|
||||
|
||||
if filepath is None:
|
||||
logger.error(
|
||||
"Firmware-Datei nicht gefunden. Geprüfte Pfade: " + ", ".join(candidates)
|
||||
)
|
||||
print("ERROR: File not found. Checked paths:\n" + "\n".join(candidates))
|
||||
return
|
||||
|
||||
try:
|
||||
# 1. Run binwalk analysis
|
||||
print("Running binwalk analysis...")
|
||||
binwalk_results = run_binwalk_analysis(filepath)
|
||||
save_binwalk_to_database(binwalk_results, raw_dataset_id)
|
||||
|
||||
# 2. Run strings extraction (EXTENDED with key-value pairing)
|
||||
print("Running strings extraction with key-value pairing...")
|
||||
strings_results = run_strings_extraction_with_pairing(filepath)
|
||||
save_strings_to_database(strings_results, raw_dataset_id)
|
||||
|
||||
# 3. Create analyzed dataset for further analysis
|
||||
create_analyzed_dataset_for_firmware(raw_dataset_id, binwalk_results, strings_results)
|
||||
|
||||
print("Firmware analysis completed successfully.")
|
||||
logger.info(f"Firmware-Analyse erfolgreich abgeschlossen: {firmware_file}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler bei Firmware-Analyse: {e}")
|
||||
print(f"ERROR during firmware analysis: {e}")
|
||||
|
||||
|
||||
def run_binwalk_analysis(filepath):
|
||||
"""
|
||||
Runs binwalk on the firmware.
|
||||
"""
|
||||
results = []
|
||||
|
||||
try:
|
||||
cmd = ['binwalk', filepath]
|
||||
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
|
||||
|
||||
lines = output.split('\n')
|
||||
|
||||
for line in lines:
|
||||
if line.startswith('DECIMAL') or line.startswith('----') or not line.strip():
|
||||
continue
|
||||
|
||||
parts = line.split(None, 2)
|
||||
if len(parts) >= 3:
|
||||
decimal = parts[0]
|
||||
hexadecimal = parts[1]
|
||||
description = parts[2]
|
||||
|
||||
results.append({
|
||||
'offset': hexadecimal,
|
||||
'description': description,
|
||||
'file_path': None
|
||||
})
|
||||
|
||||
logger.info(f"Binwalk fand {len(results)} Signaturen")
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Binwalk-Fehler: {e.output}")
|
||||
except FileNotFoundError:
|
||||
logger.error("Binwalk nicht installiert. Bitte installieren: sudo apt-get install binwalk")
|
||||
print("ERROR: binwalk not found. Install with: sudo apt-get install binwalk")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def calculate_entropy(text):
|
||||
"""Calculates the Shannon entropy of a string."""
|
||||
if not text:
|
||||
return 0
|
||||
entropy = 0
|
||||
for x in range(256):
|
||||
p_x = float(text.count(chr(x))) / len(text)
|
||||
if p_x > 0:
|
||||
entropy += - p_x * math.log(p_x, 2)
|
||||
return entropy
|
||||
|
||||
|
||||
def looks_like_secret(text):
|
||||
"""
|
||||
Checks whether a string looks like a secret (high entropy, mixed case, digits).
|
||||
"""
|
||||
if len(text) < 16 or len(text) > 256:
|
||||
return False
|
||||
|
||||
# Must contain letters AND digits
|
||||
if not (re.search(r'[a-zA-Z]', text) and re.search(r'[0-9]', text)):
|
||||
return False
|
||||
|
||||
entropy = calculate_entropy(text)
|
||||
return entropy > 3.5
|
||||
|
||||
|
||||
def is_blacklisted(text):
|
||||
"""
|
||||
Checks whether a string is on the blacklist.
|
||||
"""
|
||||
text = text.strip()
|
||||
|
||||
# Too short
|
||||
if len(text) < 3:
|
||||
return True
|
||||
|
||||
# Pure numbers
|
||||
if re.match(r'^\d+$', text):
|
||||
return True
|
||||
|
||||
# Boolean / Null
|
||||
if text.lower() in ['true', 'false', 'null', 'none', 'undefined']:
|
||||
return True
|
||||
|
||||
# Versions (standalone)
|
||||
if re.match(r'^v?\d+(\.\d+)+$', text):
|
||||
return True
|
||||
|
||||
# Only special characters
|
||||
if re.match(r'^[^a-zA-Z0-9]+$', text):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def run_strings_extraction_with_pairing(filepath, min_length=4):
|
||||
"""
|
||||
EXTENDED strings extraction with KEY-VALUE PAIRING and FILTERING.
|
||||
"""
|
||||
results = []
|
||||
|
||||
# CRITICAL KEYWORDS that expect a value in the next string
|
||||
CRITICAL_KEYS = {
|
||||
# WiFi Credentials
|
||||
'wifi-ssid': 'ssid',
|
||||
'ssid': 'ssid',
|
||||
'network-name': 'ssid',
|
||||
'ap-name': 'ssid',
|
||||
'wifi_ssid': 'ssid',
|
||||
|
||||
# WiFi Passwords
|
||||
'wifi-pass': 'password',
|
||||
'wifi-password': 'password',
|
||||
'wifi_pass': 'password',
|
||||
'wpa-psk': 'password',
|
||||
'wpa_psk': 'password',
|
||||
'pre-shared-key': 'password',
|
||||
'network-password': 'password',
|
||||
'password': 'password',
|
||||
'passwd': 'password',
|
||||
|
||||
# Location
|
||||
'location': 'location',
|
||||
'geo-location': 'location',
|
||||
'coordinates': 'location',
|
||||
'latitude': 'location',
|
||||
'longitude': 'location',
|
||||
|
||||
# Cloud Tokens
|
||||
'cloud-token': 'cloud_token',
|
||||
'cloud_token': 'cloud_token',
|
||||
'auth-token': 'cloud_token',
|
||||
'access-token': 'cloud_token',
|
||||
'bearer-token': 'cloud_token',
|
||||
'refresh-token': 'cloud_token',
|
||||
|
||||
# API Keys
|
||||
'api-key': 'api_key',
|
||||
'api_key': 'api_key',
|
||||
'apikey': 'api_key',
|
||||
'secret-key': 'api_key',
|
||||
'secret_key': 'api_key',
|
||||
'access-key': 'api_key',
|
||||
'private-key': 'api_key',
|
||||
'auth': 'api_key',
|
||||
'bearer': 'cloud_token',
|
||||
|
||||
# Device Identifiers
|
||||
'device-id': 'device_id',
|
||||
'device_id': 'device_id',
|
||||
'serial-num': 'serial',
|
||||
'serial_num': 'serial',
|
||||
'mac-address': 'mac',
|
||||
'mac_address': 'mac',
|
||||
|
||||
# Certificates
|
||||
'cert-dclrn': 'certificate',
|
||||
'certificate': 'certificate',
|
||||
}
|
||||
|
||||
# Trigger keywords for context search (simplified)
|
||||
TRIGGER_KEYWORDS = set(CRITICAL_KEYS.keys())
|
||||
|
||||
try:
|
||||
# Extract strings with offset
|
||||
cmd = ['strings', '-a', '-t', 'x', '-n', str(min_length), filepath]
|
||||
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
|
||||
|
||||
lines = output.split('\n')
|
||||
|
||||
# Statistics
|
||||
stats_total = 0
|
||||
stats_kept = 0
|
||||
stats_filtered = 0
|
||||
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i].strip()
|
||||
if not line:
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Parse: "offset string_value"
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) < 2:
|
||||
i += 1
|
||||
continue
|
||||
|
||||
offset = parts[0]
|
||||
string_value = parts[1].strip()
|
||||
stats_total += 1
|
||||
|
||||
# 1. Blacklist Check
|
||||
if is_blacklisted(string_value):
|
||||
stats_filtered += 1
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# 2. Context / key-value check
|
||||
string_lower = string_value.lower().replace('-', '_').replace(' ', '_')
|
||||
found_key_type = None
|
||||
|
||||
# Check whether the current string is a key
|
||||
for key in TRIGGER_KEYWORDS:
|
||||
if key.replace('-', '_') in string_lower:
|
||||
found_key_type = CRITICAL_KEYS.get(key)
|
||||
break
|
||||
|
||||
if found_key_type:
|
||||
# KEY found! Store key and try to find value
|
||||
results.append({
|
||||
'string_value': f"[KEY] {string_value}",
|
||||
'string_type': found_key_type
|
||||
})
|
||||
stats_kept += 1
|
||||
|
||||
# Get NEXT string as value (look ahead up to 2 strings)
|
||||
lookahead_count = 0
|
||||
j = i + 1
|
||||
while j < len(lines) and lookahead_count < 2:
|
||||
next_line = lines[j].strip()
|
||||
if next_line:
|
||||
next_parts = next_line.split(None, 1)
|
||||
if len(next_parts) >= 2:
|
||||
next_value = next_parts[1].strip()
|
||||
|
||||
# If the next value is NOT on the blacklist, we take it
|
||||
if not is_blacklisted(next_value):
|
||||
results.append({
|
||||
'string_value': f"[VALUE] {next_value}",
|
||||
'string_type': found_key_type
|
||||
})
|
||||
print(f" ✓ Found {found_key_type}: {string_value} → {next_value}")
|
||||
stats_kept += 1
|
||||
|
||||
|
||||
j += 1
|
||||
lookahead_count += 1
|
||||
|
||||
|
||||
else:
|
||||
# No key -> check whether it is a secret or an interesting category
|
||||
string_type = categorize_string(string_value)
|
||||
is_secret = looks_like_secret(string_value)
|
||||
|
||||
keep = False
|
||||
|
||||
if string_type != 'generic':
|
||||
keep = True
|
||||
elif is_secret:
|
||||
string_type = 'potential_secret'
|
||||
keep = True
|
||||
|
||||
if keep:
|
||||
results.append({
|
||||
'string_value': string_value,
|
||||
'string_type': string_type
|
||||
})
|
||||
stats_kept += 1
|
||||
else:
|
||||
stats_filtered += 1
|
||||
|
||||
i += 1
|
||||
|
||||
logger.info(f"Strings Analyse: {stats_total} total, {stats_filtered} gefiltert, {stats_kept} behalten")
|
||||
print(f"Strings Analysis: Total {stats_total} | Filtered {stats_filtered} | Kept {stats_kept}")
|
||||
|
||||
# Output statistics
|
||||
stats = {}
|
||||
for r in results:
|
||||
stats[r['string_type']] = stats.get(r['string_type'], 0) + 1
|
||||
|
||||
print("\nString-Kategorien gefunden:")
|
||||
for category, count in sorted(stats.items(), key=lambda x: x[1], reverse=True):
|
||||
print(f" {category}: {count}")
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Strings-Fehler: {e.output}")
|
||||
except FileNotFoundError:
|
||||
logger.error("strings nicht installiert. Bitte installieren: sudo apt-get install binutils")
|
||||
print("ERROR: strings not found. Install with: sudo apt-get install binutils")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def categorize_string(string_value):
|
||||
"""
|
||||
Categorizes a string based on its content.
|
||||
"""
|
||||
|
||||
|
||||
def save_binwalk_to_database(binwalk_results, raw_dataset_id):
|
||||
"""
|
||||
Stores binwalk results in the BinwalkRaw table.
|
||||
"""
|
||||
for result in binwalk_results:
|
||||
try:
|
||||
response = create_binwalk_raw(
|
||||
raw_dataset_id=raw_dataset_id,
|
||||
offset=result['offset'],
|
||||
description=result['description'],
|
||||
file_path=result.get('file_path')
|
||||
)
|
||||
|
||||
if hasattr(response, 'content'):
|
||||
response_data = json.loads(response.content.decode('utf-8'))
|
||||
if response_data['status'] != 'success':
|
||||
logger.warning(f"Binwalk-Eintrag konnte nicht gespeichert werden: {result}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Speichern von Binwalk-Daten: {e}")
|
||||
|
||||
|
||||
def save_strings_to_database(strings_results, raw_dataset_id):
|
||||
"""
|
||||
Stores ALL strings results in the StringsRaw table.
|
||||
NO LIMIT ANYMORE - all strings are stored!
|
||||
"""
|
||||
# Prioritize interesting strings for the ordering
|
||||
priority_order = ['password', 'ssid', 'cloud_token', 'api_key', 'location', 'credentials', 'certificate', 'email',
|
||||
'url', 'ip', 'mac', 'path', 'device_id', 'serial', 'generic']
|
||||
sorted_results = sorted(
|
||||
strings_results,
|
||||
key=lambda x: priority_order.index(x['string_type']) if x['string_type'] in priority_order else 999
|
||||
)
|
||||
|
||||
saved_count = 0
|
||||
for result in sorted_results:
|
||||
try:
|
||||
response = create_strings_raw(
|
||||
raw_dataset_id=raw_dataset_id,
|
||||
string_value=result['string_value'],
|
||||
string_type=result['string_type']
|
||||
)
|
||||
|
||||
if hasattr(response, 'content'):
|
||||
response_data = json.loads(response.content.decode('utf-8'))
|
||||
if response_data['status'] == 'success':
|
||||
saved_count += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Speichern von Strings-Daten: {e}")
|
||||
|
||||
logger.info(f"Strings gespeichert: {saved_count} von {len(strings_results)}")
|
||||
print(f"✓ Saved {saved_count} strings to database")
|
||||
|
||||
|
||||
def create_analyzed_dataset_for_firmware(raw_dataset_id, binwalk_results, strings_results):
|
||||
"""
|
||||
Creates an AnalyzedDataset and performs extended analysis.
|
||||
"""
|
||||
try:
|
||||
raw_dataset = RawDataset.objects.get(id=raw_dataset_id)
|
||||
file_id = raw_dataset.file.id
|
||||
|
||||
# Create/get tool
|
||||
binwalk_tool = get_or_create_tool("binwalk", "2.3.4")
|
||||
strings_tool = get_or_create_tool("strings", "2.37")
|
||||
|
||||
# Create AnalyzedDataset
|
||||
analyzed_response = create_analyzed_dataset(file_id)
|
||||
|
||||
if hasattr(analyzed_response, 'content'):
|
||||
analyzed_data = json.loads(analyzed_response.content.decode('utf-8'))
|
||||
else:
|
||||
analyzed_data = analyzed_response
|
||||
|
||||
analyzed_dataset_id = analyzed_data['data']['id']
|
||||
|
||||
# Binwalk analysis: identify important components
|
||||
analyze_binwalk_components(analyzed_dataset_id, binwalk_tool.id, binwalk_results)
|
||||
|
||||
# Strings analysis: identify critical strings (EXTENDED)
|
||||
analyze_critical_strings(analyzed_dataset_id, strings_tool.id, strings_results)
|
||||
|
||||
logger.info(f"Analyzed Dataset für Firmware erstellt: ID {analyzed_dataset_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Erstellen des Analyzed Datasets: {e}")
|
||||
|
||||
|
||||
def get_or_create_tool(tool_name, version):
|
||||
"""
|
||||
Helper function: gets an existing tool or creates a new one.
|
||||
"""
|
||||
try:
|
||||
tool = Tool.objects.get(tool_name=tool_name, version=version)
|
||||
logger.info(f"Tool gefunden: {tool_name} {version}")
|
||||
return tool
|
||||
except Tool.DoesNotExist:
|
||||
tool = Tool.objects.create(
|
||||
tool_name=tool_name,
|
||||
version=version
|
||||
)
|
||||
logger.info(f"Tool erstellt: {tool_name} {version}")
|
||||
return tool
|
||||
|
||||
|
||||
def analyze_binwalk_components(analyzed_dataset_id, tool_id, binwalk_results):
|
||||
"""
|
||||
Analyzes binwalk results and stores important components.
|
||||
"""
|
||||
important_keywords = {
|
||||
'Linux kernel': 'Linux Kernel',
|
||||
'U-Boot': 'U-Boot Bootloader',
|
||||
'filesystem': 'Filesystem',
|
||||
'certificate': 'TLS Certificate',
|
||||
'private key': 'Private Key',
|
||||
'public key': 'Public Key',
|
||||
'compressed': 'Compressed Data',
|
||||
'encrypted': 'Encrypted Data',
|
||||
'gzip': 'GZIP Archive',
|
||||
'zlib': 'ZLIB Data',
|
||||
'lzma': 'LZMA Archive',
|
||||
'squashfs': 'SquashFS Filesystem',
|
||||
'jffs2': 'JFFS2 Filesystem',
|
||||
'cramfs': 'CramFS Filesystem',
|
||||
}
|
||||
|
||||
for result in binwalk_results:
|
||||
description = result['description'].lower()
|
||||
|
||||
for keyword, component_type in important_keywords.items():
|
||||
if keyword.lower() in description:
|
||||
try:
|
||||
create_binwalk_analyzed(
|
||||
analyzed_dataset_id=analyzed_dataset_id,
|
||||
tool_id=tool_id,
|
||||
component_type=component_type,
|
||||
offset=result['offset'],
|
||||
size=None,
|
||||
entropy=None
|
||||
)
|
||||
logger.info(f"Wichtige Komponente gefunden: {component_type} @ {result['offset']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Speichern von Binwalk-Analyse: {e}")
|
||||
break
|
||||
|
||||
|
||||
def analyze_critical_strings(analyzed_dataset_id, tool_id, strings_results):
|
||||
"""
|
||||
EXTENDED analysis of critical strings with KEY-VALUE pairs.
|
||||
|
||||
Severity levels:
|
||||
- critical: passwords, cloud tokens, API keys, SSID+password combos
|
||||
- high: SSIDs, device IDs, certificates
|
||||
- medium: URLs, email addresses, location
|
||||
- low: paths, MAC addresses, serial numbers
|
||||
"""
|
||||
# Extended severity mapping
|
||||
severity_map = {
|
||||
'password': 'critical',
|
||||
'cloud_token': 'critical',
|
||||
'api_key': 'critical',
|
||||
'ssid': 'high',
|
||||
'device_id': 'high',
|
||||
'credentials': 'high',
|
||||
'certificate': 'high',
|
||||
'location': 'medium',
|
||||
'email': 'medium',
|
||||
'url': 'medium',
|
||||
'ip': 'medium',
|
||||
'serial': 'low',
|
||||
'mac': 'low',
|
||||
'path': 'low',
|
||||
'generic': 'info'
|
||||
}
|
||||
|
||||
for result in strings_results:
|
||||
string_value = result['string_value']
|
||||
string_type = result['string_type']
|
||||
|
||||
# Base severity from type
|
||||
severity = severity_map.get(string_type, 'info')
|
||||
category = string_type
|
||||
|
||||
# Store ALL critical, high and medium strings
|
||||
if severity in ['critical', 'high', 'medium']:
|
||||
try:
|
||||
create_strings_analyzed(
|
||||
analyzed_dataset_id=analyzed_dataset_id,
|
||||
tool_id=tool_id,
|
||||
string_value=string_value,
|
||||
category=category,
|
||||
severity=severity
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Speichern von Strings-Analyse: {e}")
|
||||
|
||||
|
||||
def extract_with_binwalk(filepath, output_dir):
|
||||
"""
|
||||
Optional: runs binwalk -e to extract files.
|
||||
"""
|
||||
try:
|
||||
cmd = ['binwalk', '-e', '-C', output_dir, filepath]
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
logger.info(f"Binwalk-Extraktion nach {output_dir} erfolgreich")
|
||||
return output_dir
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Binwalk-Extraktion fehlgeschlagen: {e.stderr}")
|
||||
return None
|
||||
|
||||
|
||||
def categorize_string(string_value):
|
||||
"""
|
||||
Categorizes a string based on its content.
|
||||
"""
|
||||
# URLs
|
||||
if re.search(r'https?://', string_value, re.IGNORECASE):
|
||||
return 'url'
|
||||
|
||||
# Email addresses
|
||||
if re.search(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', string_value):
|
||||
return 'email'
|
||||
|
||||
# AWS Keys
|
||||
if re.match(r'AKIA[0-9A-Z]{16}', string_value):
|
||||
return 'api_key'
|
||||
|
||||
# JWT Tokens (Basic check: 3 parts separated by dots, starts with eyJ)
|
||||
if re.match(r'eyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+', string_value):
|
||||
return 'cloud_token'
|
||||
|
||||
# Private Keys
|
||||
if 'PRIVATE KEY' in string_value:
|
||||
return 'api_key'
|
||||
|
||||
# IP addresses (IPv4)
|
||||
if re.match(r'^(?:\d{1,3}\.){3}\d{1,3}$', string_value.strip()):
|
||||
return 'ip'
|
||||
|
||||
# MAC addresses
|
||||
if re.match(r'^(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$', string_value.strip()):
|
||||
return 'mac'
|
||||
|
||||
# File paths
|
||||
if re.match(r'/(?:bin|etc|usr|var|tmp|opt|home|dev|proc|sys|lib|mnt)/', string_value):
|
||||
return 'path'
|
||||
|
||||
# Certificates
|
||||
if re.search(r'-----BEGIN [A-Z ]+-----', string_value, re.IGNORECASE):
|
||||
return 'certificate'
|
||||
|
||||
# Base64 Heuristic (length multiple of 4, valid chars, ends with = or == optionally)
|
||||
# Only if length > 20 to avoid false positives on short strings
|
||||
if len(string_value) > 20 and len(string_value) % 4 == 0 and re.match(r'^[A-Za-z0-9+/]+={0,2}$', string_value):
|
||||
return 'base64_data'
|
||||
|
||||
return 'generic'
|
||||
@@ -0,0 +1,107 @@
|
||||
import time
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from django.db import transaction
|
||||
|
||||
from IoTAnalyzer.db_create import create_analyzed_hash
|
||||
from IoTAnalyzer.models import HashRaw, HashAnalyzed, Tool, HashType, AnalyzedDataset
|
||||
|
||||
WORDLIST_PATH = "/app/django_project/IoTAnalyzer/management/commands/rockyou.txt"
|
||||
HASHCAT_PATH = "hashcat" # Update this if hashcat is in a different location
|
||||
|
||||
def crack_hashes_from_db(raw_hash_id):
|
||||
""" Crack hashes from the database using hashcat """
|
||||
|
||||
if not os.path.exists(WORDLIST_PATH):
|
||||
return {"error": f"The wordlist file {WORDLIST_PATH} does not exist."}
|
||||
|
||||
if not shutil.which(HASHCAT_PATH):
|
||||
return {"error": "Hashcat is not installed or not in PATH."}
|
||||
|
||||
results = {}
|
||||
|
||||
try:
|
||||
raw_hash = HashRaw.objects.get(id=raw_hash_id)
|
||||
hash_value = raw_hash.hash_value.strip()
|
||||
|
||||
# Identify hash type and mode
|
||||
mode_info = identify_hash_mode(hash_value)
|
||||
if mode_info is None:
|
||||
return {"error": f"Unknown hash type for hash: {hash_value}"}
|
||||
|
||||
mode, hash_type = mode_info
|
||||
print(f"Processing hash: {hash_value} (Type: {hash_type}, Mode: {mode})")
|
||||
|
||||
temp_file_path = f"temp_hash_{raw_hash_id}.txt"
|
||||
with open(temp_file_path, "w", encoding="utf-8") as temp_file:
|
||||
temp_file.write(hash_value + "\n")
|
||||
|
||||
# Run hashcat
|
||||
crack_command = [
|
||||
HASHCAT_PATH, "-m", str(mode), "-a", "0", "-D", "1,2", temp_file_path, WORDLIST_PATH, "--potfile-path", "hashcat.potfile", "--force"
|
||||
]
|
||||
|
||||
start_time = time.time()
|
||||
process = subprocess.run(crack_command, text=True, capture_output=True)
|
||||
end_time = time.time()
|
||||
duration = end_time - start_time
|
||||
crack_time_str = f"{duration:.2f}s"
|
||||
|
||||
print("Hashcat Output:", process.stdout)
|
||||
print("Hashcat Errors:", process.stderr)
|
||||
|
||||
analyzed_dataset, _ = AnalyzedDataset.objects.get_or_create(file=raw_hash.raw_dataset.file)
|
||||
tool, _ = Tool.objects.get_or_create(tool_name="Hashcat", defaults={"version": "6.x.x"})
|
||||
|
||||
# Retrieve cracked hashes
|
||||
show_command = [
|
||||
HASHCAT_PATH, "-m", str(mode), "--show", "--potfile-path", "hashcat.potfile", temp_file_path
|
||||
]
|
||||
show_process = subprocess.run(show_command, text=True, capture_output=True)
|
||||
print("Hashcat Show Output:", show_process.stdout)
|
||||
print("Hashcat Show Errors:", show_process.stderr)
|
||||
|
||||
with transaction.atomic():
|
||||
if show_process.returncode == 0 and show_process.stdout.strip():
|
||||
cracked_hashes = show_process.stdout.strip().split("\n")
|
||||
for line in cracked_hashes:
|
||||
parts = line.split(":", 1) # Split only on the first colon
|
||||
if len(parts) == 2:
|
||||
full_hash, plaintext = parts
|
||||
results[full_hash] = {"plaintext": plaintext, "type": hash_type}
|
||||
print(f"Cracked Hash: {full_hash} -> {plaintext}")
|
||||
|
||||
create_analyzed_hash(analyzed_dataset.id, tool.id, hash_type, full_hash, True, plaintext, crack_time=crack_time_str)
|
||||
|
||||
print(
|
||||
f"DEBUG: Saved hash {full_hash}, Cracked Value: {plaintext}")
|
||||
|
||||
else:
|
||||
create_analyzed_hash(analyzed_dataset.id, tool.id, hash_type, hash_value, False, crack_time=crack_time_str)
|
||||
print(f"DEBUG: Saved uncracked hash {hash_value} ")
|
||||
|
||||
os.remove(temp_file_path)
|
||||
|
||||
except HashRaw.DoesNotExist:
|
||||
return {"error": f"No HashRaw found with ID: {raw_hash_id}"}
|
||||
except FileNotFoundError:
|
||||
return {"error": "Hashcat is not installed or not in PATH."}
|
||||
except Exception as e:
|
||||
return {"error": f"Error running hashcat: {str(e)}"}
|
||||
|
||||
return results
|
||||
|
||||
def identify_hash_mode(hash_value):
|
||||
"""
|
||||
Identify hashcat mode based on the hash length and known structures.
|
||||
"""
|
||||
hash_modes = {
|
||||
32: (0, "md5"),
|
||||
40: (100, "sha1"),
|
||||
56: (300, "sha224"),
|
||||
64: (1400, "sha256"),
|
||||
96: (6100, "sha384"),
|
||||
128: (1700, "sha512")
|
||||
}
|
||||
return hash_modes.get(len(hash_value), None)
|
||||
@@ -0,0 +1,131 @@
|
||||
from scapy.all import rdpcap, Raw, DNS, DNSQR
|
||||
try:
|
||||
from scapy.layers.tls.all import TLS
|
||||
except ImportError:
|
||||
# Fallback or mock if TLS not available, though we use a try-except block in usage
|
||||
TLS = None
|
||||
import re
|
||||
import os
|
||||
from IoTAnalyzer.models import UrlRaw, HashRaw
|
||||
|
||||
|
||||
def analyze_pcap_file(pcap_file, raw_dataset_id):
|
||||
|
||||
"""
|
||||
Analyzes a PCAP file and stores found hashes and URLs in the database.
|
||||
"""
|
||||
print("Start analyzing...")
|
||||
|
||||
# Adjust the path to the file
|
||||
# Check multiple locations for the file
|
||||
possible_paths = [
|
||||
os.path.join('/app/files/uploads', pcap_file), # Mounted frontend uploads
|
||||
os.path.join('/app/files', pcap_file), # Shared files
|
||||
os.path.join('Frontend', 'iot-pre-tester-frontend', 'public', 'filesAttachment', pcap_file), # Local dev fallback
|
||||
]
|
||||
|
||||
filepath = None
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
filepath = path
|
||||
break
|
||||
|
||||
if not filepath:
|
||||
print(f"Fehler: Datei {pcap_file} nicht gefunden in: {possible_paths}")
|
||||
return
|
||||
|
||||
# Regular expressions for hashes and URLs
|
||||
hash_pattern = re.compile(r"\b[a-fA-F0-9]{32,64}\b") # MD5, SHA1, SHA256
|
||||
|
||||
# Stricter URL pattern:
|
||||
# 1. Requires http://, https://, or www. (Standard)
|
||||
# 2. OR Naked domain with AT LEAST 2 dots (e.g. sub.domain.com) to catch cloud URLs but avoid "J.mINT"
|
||||
url_pattern = re.compile(
|
||||
r"\b(?:"
|
||||
r"(?:https?://|www\.)(?:[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6})" # Protocol/www required
|
||||
r"|"
|
||||
r"(?:[a-zA-Z0-9-]+\.){2,}[a-zA-Z]{2,6}" # Naked domain with >= 2 dots
|
||||
r"|"
|
||||
r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" # IPv4
|
||||
r")"
|
||||
r"(?:/[a-zA-Z0-9._%/-]*)?\b"
|
||||
)
|
||||
|
||||
# Load PCAP file
|
||||
try:
|
||||
packets = rdpcap(filepath)
|
||||
except Exception as e:
|
||||
print(f"Fehler beim Öffnen der PCAP-Datei: {e}")
|
||||
return
|
||||
|
||||
found_hashes = set()
|
||||
found_urls = set()
|
||||
|
||||
# Analyze packets
|
||||
for pkt in packets:
|
||||
# 1. DNS Queries (Reliable source for domains)
|
||||
if pkt.haslayer(DNS) and pkt[DNS].qd:
|
||||
try:
|
||||
qname = pkt[DNS].qd.qname.decode('utf-8', errors='ignore')
|
||||
if qname.endswith('.'):
|
||||
qname = qname[:-1]
|
||||
if qname:
|
||||
found_urls.add(qname)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. TLS Server Name Indication (SNI)
|
||||
# Attempt to extract SNI if TLS layer is present
|
||||
if TLS and pkt.haslayer(TLS):
|
||||
try:
|
||||
# Iterate through TLS messages to find ClientHello and extensions
|
||||
if hasattr(pkt[TLS], 'msg'):
|
||||
for msg in pkt[TLS].msg:
|
||||
if hasattr(msg, 'ext'):
|
||||
for ext in msg.ext:
|
||||
if hasattr(ext, 'servernames'):
|
||||
for servername in ext.servernames:
|
||||
if hasattr(servername, 'servername'):
|
||||
sni = servername.servername.decode('utf-8', errors='ignore')
|
||||
if sni:
|
||||
found_urls.add(sni)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. Raw Payload (Fallback with strict regex)
|
||||
if pkt.haslayer(Raw): # Check whether the packet has a payload
|
||||
payload = pkt[Raw].load.decode(errors='ignore')
|
||||
|
||||
# Search for hashes
|
||||
found_hashes.update(hash_pattern.findall(payload))
|
||||
|
||||
# Search for URLs
|
||||
found_urls.update(url_pattern.findall(payload))
|
||||
|
||||
# Store results in the database
|
||||
save_urls_to_database(found_urls, raw_dataset_id)
|
||||
save_hashes_to_database(found_hashes, raw_dataset_id)
|
||||
|
||||
print("Done analyzing.")
|
||||
|
||||
|
||||
def save_urls_to_database(urls, raw_dataset_id):
|
||||
"""
|
||||
Stores extracted URLs in the database.
|
||||
"""
|
||||
for url in urls:
|
||||
UrlRaw.objects.create(
|
||||
raw_dataset_id=raw_dataset_id,
|
||||
url=url
|
||||
)
|
||||
|
||||
|
||||
def save_hashes_to_database(hashes, raw_dataset_id):
|
||||
"""
|
||||
Stores extracted hashes in the database.
|
||||
"""
|
||||
for hash_value in hashes:
|
||||
HashRaw.objects.create(
|
||||
raw_dataset_id=raw_dataset_id,
|
||||
hash_value=hash_value
|
||||
)
|
||||
@@ -0,0 +1,212 @@
|
||||
import dns.resolver
|
||||
import requests
|
||||
import ssl
|
||||
import socket
|
||||
import nmap
|
||||
from django.db import transaction
|
||||
import os
|
||||
import re
|
||||
|
||||
from IoTAnalyzer.db_create import create_analyzed_url
|
||||
from IoTAnalyzer.models import Tool, TlsVersion, CipherSuite, UrlRaw, AnalyzedDataset
|
||||
from urllib.parse import urlparse
|
||||
from datetime import datetime
|
||||
|
||||
LOG_FILE = "errors.log" # Log file for errors
|
||||
|
||||
|
||||
def log_error(message):
|
||||
"""
|
||||
Logs an error message to the log file with a timestamp.
|
||||
"""
|
||||
with open(LOG_FILE, "a") as log:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
log.write(f"[{timestamp}] {message}\n")
|
||||
|
||||
|
||||
def validate_url(url):
|
||||
"""
|
||||
Validates if a given string is a properly formatted URL or domain.
|
||||
"""
|
||||
try:
|
||||
result = urlparse(url)
|
||||
return result.netloc or result.path
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def resolve_dns(query):
|
||||
"""
|
||||
Resolves a DNS query and returns a list of resolved IPs.
|
||||
"""
|
||||
try:
|
||||
result = dns.resolver.resolve(query, 'A')
|
||||
resolved_ips = [str(rdata) for rdata in result]
|
||||
if not resolved_ips:
|
||||
log_error(f"No IPs resolved for DNS query: {query}")
|
||||
return resolved_ips
|
||||
except dns.resolver.NXDOMAIN:
|
||||
log_error(f"NXDOMAIN: No such domain {query}")
|
||||
except dns.resolver.Timeout:
|
||||
log_error(f"Timeout: DNS query for {query} timed out")
|
||||
except dns.resolver.NoNameservers:
|
||||
log_error(f"NoNameservers: No name servers found for {query}")
|
||||
except Exception as e:
|
||||
log_error(f"General DNS error for {query}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def scan_ports(target):
|
||||
"""
|
||||
Scans the given IP for open ports using Nmap.
|
||||
Returns a list of open ports.
|
||||
"""
|
||||
nm = nmap.PortScanner()
|
||||
open_ports = []
|
||||
|
||||
try:
|
||||
nm.scan(hosts=target, arguments='-sT -Pn -p 22,80,443,8000,8080 --scan-delay 500ms --max-retries 1')
|
||||
#nm.scan(hosts=ip, arguments='-sS -Pn -F')
|
||||
if target in nm.all_hosts():
|
||||
for port in nm[target].all_tcp():
|
||||
if 'open' in nm[target]['tcp'][port]['state']:
|
||||
open_ports.append(port)
|
||||
except Exception as e:
|
||||
log_error(f"Error scanning IP {target}: {e}")
|
||||
|
||||
if not open_ports:
|
||||
print("Python-Nmap failed, using OS-level Nmap...")
|
||||
os.system(f"nmap -sT -Pn -p 22,80,443,8000,8080 {target} > nmap_output.txt")
|
||||
with open("nmap_output.txt", "r") as f:
|
||||
output = f.read()
|
||||
if "open" in output:
|
||||
open_ports = []
|
||||
for port in [22, 80, 443, 8000, 8080]:
|
||||
pattern = rf"^{port}/tcp\s+open\s"
|
||||
if re.search(pattern, output, re.MULTILINE):
|
||||
open_ports.append(port)
|
||||
|
||||
return open_ports
|
||||
|
||||
|
||||
def scan_ssl(target):
|
||||
"""
|
||||
Performs an SSL scan on the given domain or IP.
|
||||
Retrieves TLS version, cipher suite, and certificate presence.
|
||||
"""
|
||||
context = ssl.create_default_context()
|
||||
context.check_hostname = True
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
context.verify_mode = ssl.CERT_REQUIRED
|
||||
context.load_default_certs()
|
||||
context.set_alpn_protocols(["http/1.1", "h2"])
|
||||
context.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 # Disable old protocols
|
||||
|
||||
scan_result = {
|
||||
'tls_version': None,
|
||||
'cipher': None,
|
||||
'certificate': None,
|
||||
}
|
||||
|
||||
try:
|
||||
with socket.create_connection((target, 443), timeout=5) as sock:
|
||||
with context.wrap_socket(sock, server_hostname=target) as ssock:
|
||||
cert = ssock.getpeercert()
|
||||
cipher = ssock.cipher()
|
||||
tls_version = ssock.version()
|
||||
|
||||
scan_result['certificate'] = 'Present' if cert else 'Not Present'
|
||||
scan_result['cipher'] = cipher[0] if cipher else None
|
||||
scan_result['tls_version'] = tls_version
|
||||
except ssl.SSLError as e:
|
||||
log_error(f"SSL Error during scan for {target}: {e}")
|
||||
except socket.timeout:
|
||||
log_error(f"Timeout during SSL scan for {target}")
|
||||
except Exception as e:
|
||||
log_error(f"General error during SSL scan for {target}: {e}")
|
||||
|
||||
return scan_result
|
||||
|
||||
|
||||
def fetch_important_headers(url):
|
||||
"""
|
||||
Fetches only important headers (e.g., Server, Content-Type).
|
||||
"""
|
||||
try:
|
||||
response = requests.head(url, timeout=5)
|
||||
headers = response.headers
|
||||
important_headers = {
|
||||
"Server": headers.get("Server", "N/A"),
|
||||
"Content-Type": headers.get("Content-Type", "N/A"),
|
||||
"Strict-Transport-Security": headers.get("Strict-Transport-Security", "N/A"),
|
||||
}
|
||||
return important_headers
|
||||
except Exception as e:
|
||||
log_error(f"Error fetching headers for {url}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def analyze_target(query, targets, analyzed_dataset):
|
||||
"""
|
||||
Ensures each URL is analyzed only once.
|
||||
Combines SSL, HTTP headers, and port scanning for a DNS query and its targets.
|
||||
Saves the results into the database with detailed logging.
|
||||
"""
|
||||
target = query.replace("http://", "").replace("https://", "").split('/')[0]
|
||||
ssl_info = scan_ssl(target)
|
||||
headers = fetch_important_headers(f"https://{target}")
|
||||
open_ports = scan_ports(target)
|
||||
|
||||
print(f"Analyzing URL: {query}")
|
||||
print(f"SSL Info: {ssl_info}")
|
||||
print(f"Headers: {headers}")
|
||||
print(f"Open Ports: {open_ports}")
|
||||
|
||||
with transaction.atomic():
|
||||
tool, _ = Tool.objects.get_or_create(tool_name="DNS Analyzer", version="1.0")
|
||||
|
||||
if ssl_info['tls_version'] == None:
|
||||
tls_version, _ = TlsVersion.objects.get_or_create(tls_version="N/A")
|
||||
else:
|
||||
tls_version = ssl_info['tls_version']
|
||||
|
||||
if ssl_info['cipher'] == None:
|
||||
cipher, _ = CipherSuite.objects.get_or_create(cipher_suite="N/A")
|
||||
else:
|
||||
cipher = ssl_info['cipher']
|
||||
|
||||
create_analyzed_url(analyzed_dataset.id, tool.id, query, open_ports, [tls_version], [cipher])
|
||||
|
||||
log_error(f"Data saved for URL: {query}")
|
||||
|
||||
|
||||
|
||||
def get_urls_from_db(raw_url_id):
|
||||
"""
|
||||
Main function to handle the URL analysis for a given raw_url_id.
|
||||
"""
|
||||
try:
|
||||
raw_url = UrlRaw.objects.get(id=raw_url_id)
|
||||
except UrlRaw.DoesNotExist:
|
||||
log_error(f"Raw URL with ID {raw_url_id} not found.")
|
||||
return
|
||||
|
||||
queries = [raw_url.url]
|
||||
analysis_targets = {}
|
||||
|
||||
for query in queries:
|
||||
if not validate_url(query):
|
||||
log_error(f"Invalid URL: {query}")
|
||||
continue
|
||||
|
||||
ips = resolve_dns(query)
|
||||
analysis_targets[query] = ips if ips else [query]
|
||||
|
||||
# analyzed_dataset = raw_url.raw_dataset.file.analyzeddataset_set.last()
|
||||
analyzed_dataset, _ = AnalyzedDataset.objects.get_or_create(file=raw_url.raw_dataset.file)
|
||||
if not analyzed_dataset:
|
||||
log_error(f"No analyzed dataset found for the file associated with URL: {raw_url.url}")
|
||||
return
|
||||
|
||||
for query, targets in analysis_targets.items():
|
||||
analyze_target(query, targets, analyzed_dataset)
|
||||
Reference in New Issue
Block a user