Home /Modules/server-edition/log-consumer.sh

log-consumer.sh

server-edition/port-awareness/log-consumer.sh · 50 line(s) · bash

#!/usr/bin/env sh
# SLeeLa privacy-preserving traffic-log consumer.
# Produces dated message.digest files and consumes the source log daily or at 50 MiB.
set -eu
STATE=${SLEELA_SERVER_STATE:-server-edition/state}
TRAFFIC_DIR=${SLEELA_TRAFFIC_LOG_DIR:-$STATE/traffic}
OUT_DIR=${SLEELA_MESSAGE_DIGEST_DIR:-$STATE/message-digest}
LOGFILE=$TRAFFIC_DIR/traffic.log
mkdir -p "$TRAFFIC_DIR" "$OUT_DIR"
MAX_BYTES=${SLEELA_LOG_MAX_BYTES:-52428800}
STAMP=$(date -u +%Y-%m-%dT%H-%M-%SZ 2>/dev/null || echo now)
DAY=$(date -u +%Y-%m-%d 2>/dev/null || echo unknown-date)
HOUR=$(date -u +%H-%M-%S 2>/dev/null || echo unknown-time)
TARGET="$OUT_DIR/$DAY/$HOUR/message.digest"
should_consume=false
if [ -f "$LOGFILE" ]; then
  size=$(wc -c < "$LOGFILE" | tr -d ' ')
  [ "${size:-0}" -ge "$MAX_BYTES" ] && should_consume=true
  if [ -n "${SLEELA_LAST_LOG_DAY:-}" ] && [ "${SLEELA_LAST_LOG_DAY}" != "$DAY" ]; then should_consume=true; fi
  [ "${SLEELA_FORCE_LOG_CONSUME:-false}" = true ] && should_consume=true
fi
[ "$should_consume" = true ] || exit 0
mkdir -p "$(dirname "$TARGET")"
{
  printf 'schema=sleela.message.digest.v1 timestamp=%s\n' "$STAMP"
  printf 'privacy=ip-redacted national-data-redacted contact-data-redacted payload-redacted\n'
  printf 'source_bytes=%s\n' "${size:-0}"
  if [ -s "$LOGFILE" ]; then
    # Keep protocol/port/event structure while removing addresses, identity/contact fields,
    # and packet payload. This digest intentionally cannot reconstruct a remote identity.
    sed -E \
      -e 's/([0-9A-Fa-f]{0,4}:){2,}[0-9A-Fa-f:]+/[IP-REDACTED]/g' \
      -e 's/[0-9]{1,3}(\.[0-9]{1,3}){3}/[IP-REDACTED]/g' \
      -e 's/([[:space:]]|^)(National|Citizen|Bank|Banker|Cromsmrus|Contact|Email|Phone|Address|Identity)[_-]?[A-Za-z0-9.-]*=[^[:space:]]+/\1[DATA-REDACTED]/gI' \
      -e 's/([[:space:]]|^)([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}([[:space:]]|$)/\1[MAC-REDACTED]\3/g' \
      -e 's/0x[0-9A-Fa-f]+/[PAYLOAD-REDACTED]/g' \
      -e 's/[[:space:]]+([0-9A-Fa-f]{2}[[:space:]]+){8,}.*/ [PAYLOAD-REDACTED]/g' \
      "$LOGFILE" | awk '
        /IP-REDACTED|DATA-REDACTED|MAC-REDACTED|PAYLOAD-REDACTED|traffic_logger=/ {
          gsub(/[[:space:]]+/, " ");
          print
        }
      '
  fi
} > "$TARGET"
# Atomic consume: the raw packet log is no longer retained after successful digest creation.
: > "$LOGFILE"
printf '%s consumed source=traffic.log target=%s\n' "$STAMP" "$TARGET" >> "$TRAFFIC_DIR/consumer.log"
printf '%s\n' "$DAY" > "$TRAFFIC_DIR/last-consumed-day"