Home /Modules/antivirus/sleelavirin_import_clamav.py

sleelavirin_import_clamav.py

antivirus/tools/sleelavirin_import_clamav.py · 245 line(s) · py

#!/usr/bin/env python3
"""Import only compatible whole-file SHA-256 records from extracted ClamAV .hsb text.

This is not a CVD/CLD decoder and never modifies the active index when input
cannot produce a valid non-empty candidate database.
"""
import argparse
import hashlib
import json
import os
import shutil
from pathlib import Path
import re
import sys
import tempfile
from datetime import datetime, timezone

SHA256 = re.compile(r"^[0-9a-fA-F]{64}$")


def validate_database(path):
    errors, count = [], 0
    try:
        with path.open("r", encoding="utf-8") as handle:
            for number, raw in enumerate(handle, 1):
                line = raw.strip()
                if not line or line.startswith("#"):
                    continue
                digest = line.split(None, 1)[0]
                if not SHA256.fullmatch(digest):
                    errors.append(f"{path}:{number}: expected a 64-character SHA-256 digest")
                else:
                    count += 1
    except OSError as exc:
        return [f"cannot read {path}: {exc}"], 0
    return errors, count


def validate_manifest(index_path, manifest_path):
    """Verify manifest provenance fields and bind the manifest to exact index bytes."""
    try:
        raw = index_path.read_bytes()
        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
        return [f"cannot read/parse manifest or index: {exc}"]
    if not isinstance(manifest, dict):
        return ["manifest root must be a JSON object"]
    errors = []
    if manifest.get("schema") != "sleelavirin-native-index-manifest-v1":
        errors.append("unsupported or missing manifest schema")
    actual_digest = hashlib.sha256(raw).hexdigest()
    if manifest.get("index_sha256") != actual_digest:
        errors.append("manifest index_sha256 does not match active index bytes")
    _, count = validate_database(index_path)
    try:
        if int(manifest.get("signature_count", -1)) != count:
            errors.append("manifest signature_count does not match active index")
    except (TypeError, ValueError):
        errors.append("manifest signature_count is invalid")
    return errors


def collect_signatures(source):
    entries, skipped = {}, 0
    paths = sorted(p for p in source.rglob("*") if p.is_file() and p.suffix.lower() == ".hsb")
    if not paths:
        return entries, skipped, [f"no extracted .hsb files found under {source}"]
    errors = []
    for path in paths:
        try:
            lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
        except (OSError, UnicodeError) as exc:
            errors.append(f"cannot read {path}: {exc}")
            continue
        for line_no, raw in enumerate(lines, 1):
            line = raw.strip()
            if not line or line.startswith("#"):
                continue
            fields = line.split(":")
            if len(fields) < 3 or not SHA256.fullmatch(fields[0].strip()):
                skipped += 1
                continue
            try:
                size = int(fields[1])
                if size < 0:
                    raise ValueError()
            except ValueError:
                skipped += 1
                continue
            label = ":".join(fields[2:]).strip() or "clamav-import"
            entries.setdefault(fields[0].lower(), set()).add(label)
    if errors:
        return entries, skipped, errors
    if not entries:
        errors.append("no compatible whole-file SHA-256 .hsb records found")
    return entries, skipped, errors


def stage_text(path, content):
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, temp_name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
    temp_path = Path(temp_name)
    try:
        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
            handle.write(content)
            handle.flush()
            os.fsync(handle.fileno())
        return temp_path
    except Exception:
        try:
            temp_path.unlink()
        except OSError:
            pass
        raise


def main():
    parser = argparse.ArgumentParser(description="Import compatible extracted ClamAV SHA-256 .hsb records.")
    parser.add_argument("--source", type=Path, help="directory of extracted plain-text .hsb files")
    parser.add_argument("--output", type=Path, help="native SHA-256 index destination")
    parser.add_argument("--check", type=Path, help="validate an existing native index")
    parser.add_argument("--check-manifest", type=Path, help="also verify a provenance manifest against --check index bytes")
    parser.add_argument("--source-version", default="unspecified", help="ClamAV database/version identifier")
    parser.add_argument("--source-url", default="", help="origin URL or internal artifact identifier")
    parser.add_argument("--manifest", type=Path, help="provenance JSON path (default: OUTPUT.manifest.json)")
    args = parser.parse_args()
    if args.check:
        errors, count = validate_database(args.check)
        if args.check_manifest:
            errors.extend(validate_manifest(args.check, args.check_manifest))
        for error in errors:
            print(f"INVALID {error}", file=sys.stderr)
        if errors:
            return 2
        print(f"VALID {args.check}: {count} signature(s)")
        if args.check_manifest:
            print(f"VALID MANIFEST {args.check_manifest}")
        return 0
    if args.check_manifest:
        parser.error("--check-manifest requires --check")
    if not args.source or not args.output:
        parser.error("provide --source and --output, or use --check")
    source = args.source.expanduser()
    output = args.output.expanduser()
    if not source.is_dir():
        parser.error("source must be a directory of extracted text signature files")
    try:
        entries, skipped, errors = collect_signatures(source)
    except OSError as exc:
        print(f"ERROR reading source tree: {exc}", file=sys.stderr)
        return 2
    if errors:
        for error in errors:
            print(f"ERROR: {error}", file=sys.stderr)
        print("Active index was not changed.", file=sys.stderr)
        return 2

    lines = [
        "# Imported compatible whole-file SHA-256 records from extracted ClamAV .hsb files.",
        "# This subset does not represent all ClamAV detections.",
    ]
    for digest in sorted(entries):
        lines.append(f"{digest} {', '.join(sorted(entries[digest]))}")
    candidate = "\n".join(lines) + "\n"
    # Validate a same-directory temporary candidate before replacing the active index.
    output.parent.mkdir(parents=True, exist_ok=True)
    fd, temp_name = tempfile.mkstemp(prefix=output.name + ".", suffix=".candidate", dir=str(output.parent))
    temp_path = Path(temp_name)
    try:
        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
            handle.write(candidate)
            handle.flush()
            os.fsync(handle.fileno())
        errors, count = validate_database(temp_path)
        if errors or count != len(entries):
            for error in errors:
                print(f"ERROR validating candidate: {error}", file=sys.stderr)
            print("Candidate index invalid; active index was not changed.", file=sys.stderr)
            return 2
        digest = hashlib.sha256(candidate.encode("utf-8")).hexdigest()
        manifest_path = args.manifest.expanduser() if args.manifest else Path(str(output) + ".manifest.json")
        if manifest_path.resolve() == output.resolve():
            print("ERROR: manifest path must differ from the active index path", file=sys.stderr)
            return 2
        manifest = {
            "schema": "sleelavirin-native-index-manifest-v1",
            "source_engine": "ClamAV",
            "source_version": args.source_version,
            "source_url": args.source_url or None,
            "source_directory": str(source.resolve()),
            "imported_at_utc": datetime.now(timezone.utc).isoformat(),
            "format": "native whole-file SHA-256 digest plus optional label",
            "signature_count": count,
            "skipped_record_count": skipped,
            "index_sha256": digest,
            "coverage_note": "Compatible extracted .hsb subset only; not equivalent to the full ClamAV signature database.",
        }
        # Stage provenance before changing the active index. If publication fails,
        # restore the previous index (or remove the new index if none existed).
        manifest_temp = stage_text(manifest_path, json.dumps(manifest, indent=2, sort_keys=True) + "\n")
        backup_path = None
        had_output = output.exists()
        if had_output:
            fd_backup, backup_name = tempfile.mkstemp(prefix=output.name + ".", suffix=".backup", dir=str(output.parent))
            os.close(fd_backup)
            backup_path = Path(backup_name)
            shutil.copy2(output, backup_path)
        try:
            os.replace(temp_path, output)
            os.replace(manifest_temp, manifest_path)
        except OSError:
            if backup_path is not None and backup_path.exists():
                os.replace(backup_path, output)
            elif not had_output:
                try:
                    output.unlink()
                except OSError:
                    pass
            raise
        finally:
            for temporary in (manifest_temp, backup_path):
                if temporary is not None:
                    try:
                        temporary.unlink()
                    except OSError:
                        pass
    except OSError as exc:
        print(f"ERROR writing candidate/index manifest: {exc}", file=sys.stderr)
        print("Active index is preserved or rolled back where the filesystem permits.", file=sys.stderr)
        return 2
    finally:
        try:
            temp_path.unlink()
        except OSError:
            pass
    print(f"IMPORTED {count} unique SHA-256 signatures -> {output}")
    print(f"PROVENANCE {manifest_path}")
    print(f"SKIPPED {skipped} unsupported/invalid records")
    print("NOTE: this is a compatible subset, not a complete ClamAV database.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())