sleelavirin_import_clamav.py
antivirus/tools/sleelavirin_import_clamav.py · 245 line(s) · py
#!/usr/bin/env python3
"""Import only compatible whole-file SHA-256 records from extracted ClamAV .hsb text.
This is not a CVD/CLD decoder and never modifies the active index when input
cannot produce a valid non-empty candidate database.
"""
import argparse
import hashlib
import json
import os
import shutil
from pathlib import Path
import re
import sys
import tempfile
from datetime import datetime, timezone
SHA256 = re.compile(r"^[0-9a-fA-F]{64}$")
def validate_database(path):
errors, count = [], 0
try:
with path.open("r", encoding="utf-8") as handle:
for number, raw in enumerate(handle, 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
digest = line.split(None, 1)[0]
if not SHA256.fullmatch(digest):
errors.append(f"{path}:{number}: expected a 64-character SHA-256 digest")
else:
count += 1
except OSError as exc:
return [f"cannot read {path}: {exc}"], 0
return errors, count
def validate_manifest(index_path, manifest_path):
"""Verify manifest provenance fields and bind the manifest to exact index bytes."""
try:
raw = index_path.read_bytes()
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
return [f"cannot read/parse manifest or index: {exc}"]
if not isinstance(manifest, dict):
return ["manifest root must be a JSON object"]
errors = []
if manifest.get("schema") != "sleelavirin-native-index-manifest-v1":
errors.append("unsupported or missing manifest schema")
actual_digest = hashlib.sha256(raw).hexdigest()
if manifest.get("index_sha256") != actual_digest:
errors.append("manifest index_sha256 does not match active index bytes")
_, count = validate_database(index_path)
try:
if int(manifest.get("signature_count", -1)) != count:
errors.append("manifest signature_count does not match active index")
except (TypeError, ValueError):
errors.append("manifest signature_count is invalid")
return errors
def collect_signatures(source):
entries, skipped = {}, 0
paths = sorted(p for p in source.rglob("*") if p.is_file() and p.suffix.lower() == ".hsb")
if not paths:
return entries, skipped, [f"no extracted .hsb files found under {source}"]
errors = []
for path in paths:
try:
lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
except (OSError, UnicodeError) as exc:
errors.append(f"cannot read {path}: {exc}")
continue
for line_no, raw in enumerate(lines, 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
fields = line.split(":")
if len(fields) < 3 or not SHA256.fullmatch(fields[0].strip()):
skipped += 1
continue
try:
size = int(fields[1])
if size < 0:
raise ValueError()
except ValueError:
skipped += 1
continue
label = ":".join(fields[2:]).strip() or "clamav-import"
entries.setdefault(fields[0].lower(), set()).add(label)
if errors:
return entries, skipped, errors
if not entries:
errors.append("no compatible whole-file SHA-256 .hsb records found")
return entries, skipped, errors
def stage_text(path, content):
path.parent.mkdir(parents=True, exist_ok=True)
fd, temp_name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
temp_path = Path(temp_name)
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
handle.write(content)
handle.flush()
os.fsync(handle.fileno())
return temp_path
except Exception:
try:
temp_path.unlink()
except OSError:
pass
raise
def main():
parser = argparse.ArgumentParser(description="Import compatible extracted ClamAV SHA-256 .hsb records.")
parser.add_argument("--source", type=Path, help="directory of extracted plain-text .hsb files")
parser.add_argument("--output", type=Path, help="native SHA-256 index destination")
parser.add_argument("--check", type=Path, help="validate an existing native index")
parser.add_argument("--check-manifest", type=Path, help="also verify a provenance manifest against --check index bytes")
parser.add_argument("--source-version", default="unspecified", help="ClamAV database/version identifier")
parser.add_argument("--source-url", default="", help="origin URL or internal artifact identifier")
parser.add_argument("--manifest", type=Path, help="provenance JSON path (default: OUTPUT.manifest.json)")
args = parser.parse_args()
if args.check:
errors, count = validate_database(args.check)
if args.check_manifest:
errors.extend(validate_manifest(args.check, args.check_manifest))
for error in errors:
print(f"INVALID {error}", file=sys.stderr)
if errors:
return 2
print(f"VALID {args.check}: {count} signature(s)")
if args.check_manifest:
print(f"VALID MANIFEST {args.check_manifest}")
return 0
if args.check_manifest:
parser.error("--check-manifest requires --check")
if not args.source or not args.output:
parser.error("provide --source and --output, or use --check")
source = args.source.expanduser()
output = args.output.expanduser()
if not source.is_dir():
parser.error("source must be a directory of extracted text signature files")
try:
entries, skipped, errors = collect_signatures(source)
except OSError as exc:
print(f"ERROR reading source tree: {exc}", file=sys.stderr)
return 2
if errors:
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
print("Active index was not changed.", file=sys.stderr)
return 2
lines = [
"# Imported compatible whole-file SHA-256 records from extracted ClamAV .hsb files.",
"# This subset does not represent all ClamAV detections.",
]
for digest in sorted(entries):
lines.append(f"{digest} {', '.join(sorted(entries[digest]))}")
candidate = "\n".join(lines) + "\n"
# Validate a same-directory temporary candidate before replacing the active index.
output.parent.mkdir(parents=True, exist_ok=True)
fd, temp_name = tempfile.mkstemp(prefix=output.name + ".", suffix=".candidate", dir=str(output.parent))
temp_path = Path(temp_name)
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
handle.write(candidate)
handle.flush()
os.fsync(handle.fileno())
errors, count = validate_database(temp_path)
if errors or count != len(entries):
for error in errors:
print(f"ERROR validating candidate: {error}", file=sys.stderr)
print("Candidate index invalid; active index was not changed.", file=sys.stderr)
return 2
digest = hashlib.sha256(candidate.encode("utf-8")).hexdigest()
manifest_path = args.manifest.expanduser() if args.manifest else Path(str(output) + ".manifest.json")
if manifest_path.resolve() == output.resolve():
print("ERROR: manifest path must differ from the active index path", file=sys.stderr)
return 2
manifest = {
"schema": "sleelavirin-native-index-manifest-v1",
"source_engine": "ClamAV",
"source_version": args.source_version,
"source_url": args.source_url or None,
"source_directory": str(source.resolve()),
"imported_at_utc": datetime.now(timezone.utc).isoformat(),
"format": "native whole-file SHA-256 digest plus optional label",
"signature_count": count,
"skipped_record_count": skipped,
"index_sha256": digest,
"coverage_note": "Compatible extracted .hsb subset only; not equivalent to the full ClamAV signature database.",
}
# Stage provenance before changing the active index. If publication fails,
# restore the previous index (or remove the new index if none existed).
manifest_temp = stage_text(manifest_path, json.dumps(manifest, indent=2, sort_keys=True) + "\n")
backup_path = None
had_output = output.exists()
if had_output:
fd_backup, backup_name = tempfile.mkstemp(prefix=output.name + ".", suffix=".backup", dir=str(output.parent))
os.close(fd_backup)
backup_path = Path(backup_name)
shutil.copy2(output, backup_path)
try:
os.replace(temp_path, output)
os.replace(manifest_temp, manifest_path)
except OSError:
if backup_path is not None and backup_path.exists():
os.replace(backup_path, output)
elif not had_output:
try:
output.unlink()
except OSError:
pass
raise
finally:
for temporary in (manifest_temp, backup_path):
if temporary is not None:
try:
temporary.unlink()
except OSError:
pass
except OSError as exc:
print(f"ERROR writing candidate/index manifest: {exc}", file=sys.stderr)
print("Active index is preserved or rolled back where the filesystem permits.", file=sys.stderr)
return 2
finally:
try:
temp_path.unlink()
except OSError:
pass
print(f"IMPORTED {count} unique SHA-256 signatures -> {output}")
print(f"PROVENANCE {manifest_path}")
print(f"SKIPPED {skipped} unsupported/invalid records")
print("NOTE: this is a compatible subset, not a complete ClamAV database.")
return 0
if __name__ == "__main__":
raise SystemExit(main())