#!/usr/bin/env sh
# test-abstraction-db.sh
#
# Cross-check for spine's two serialisations of the document abstraction:
# the .ssp text file and the <doc>.ocda.db sqlite database.
#
# It generates both for every sample document and asserts that they agree
# object for object and field for field. There is no committed reference
# set: the two artefacts check each other, which is what catches the thing
# that went wrong before, one writer gaining a field and the other not.
#
# Usage:
#   SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \
#     ./test/test-abstraction-db.sh ./bin/spine-ldc
#
# Exit codes:
#   0  the two agree for every document
#   1  disagreements found (printed)
#   2  set-up problem (no $SpinePOD, no binary, no sqlite3, no samples)

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"

if [ -z "$SpinePOD" ]; then
  echo "ERROR: \$SpinePOD is not set." >&2
  echo "  Set it to the sample pod directory, relative to $SPINE_DIR" >&2
  echo "  e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2
  exit 2
fi

SAMPLES_RAW="$SPINE_DIR/$SpinePOD"
if [ -d "$SAMPLES_RAW" ]; then
  SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)"
else
  SAMPLES_DIR="$SAMPLES_RAW"
fi

SPINE_BIN="${1:-}"
if [ -z "$SPINE_BIN" ]; then
  if   [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine"
  elif [ -x "$SPINE_DIR/bin/spine-ldc" ];    then SPINE_BIN="$SPINE_DIR/bin/spine-ldc"
  elif [ -x "$SPINE_DIR/bin/spine" ];        then SPINE_BIN="$SPINE_DIR/bin/spine"
  else
    echo "ERROR: spine binary not found. Specify path as argument." >&2
    exit 2
  fi
fi

if ! command -v sqlite3 > /dev/null 2>&1; then
  echo "ERROR: sqlite3 not on PATH." >&2
  echo "  e.g. nix shell \"nixpkgs#sqlite\" -c ./test/test-abstraction-db.sh $SPINE_BIN" >&2
  exit 2
fi

if [ ! -d "$SAMPLES_DIR" ]; then
  echo "ERROR: sample documents not found at $SAMPLES_DIR" >&2
  exit 2
fi

OUT_DIR="$SCRIPT_DIR/current-db"

echo "spine binary: $SPINE_BIN"
echo "samples: $SAMPLES_DIR"
echo "Generating .ssp and .ocda.db ..."
rm -rf "$OUT_DIR"
mkdir -p "$OUT_DIR"
$SPINE_BIN --show-abstraction --ocda-db --skip-output \
  --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1

FAILURES=0
DOCS=0

# ssp property -> sqlite predicate counting the same thing
#   "<property>|<predicate>"
CHECKS="
children|children IS NOT NULL
ancestors|ancestors IS NOT NULL
ancestors_collapsed|ancestors_collapsed IS NOT NULL
dom_status|dom_status IS NOT NULL
dom_status_collapsed|dom_status_collapsed IS NOT NULL
last_descendant|last_descendant_ocn != 0
parent_lev|parent_lev != 0
heading_lev_collapsed|heading_lev_collapsed IS NOT NULL
section|is_of_section IS NOT NULL AND is_of_section != section
part|is_of_part IS NOT NULL
is_of_type|is_of_type IS NOT NULL
attrib|attrib IS NOT NULL
meta_lang|meta_lang IS NOT NULL
meta_syntax|meta_syntax IS NOT NULL
sha256|sha256 IS NOT NULL
heading_ancestors_text|heading_ancestors_text IS NOT NULL
segment|segment IS NOT NULL
segment_prev|segment_prev IS NOT NULL
segment_next|segment_next IS NOT NULL
segment_html_is|segment_html_is IS NOT NULL
segment_epub_is|segment_epub_is IS NOT NULL
segment_epub|segment_epub IS NOT NULL
anchor|anchor IS NOT NULL AND anchor != segment
heading_lev_anchor|heading_lev_anchor IS NOT NULL
table_cols|table_cols IS NOT NULL
table_widths|table_widths IS NOT NULL
table_aligns|table_aligns IS NOT NULL
dummy: true|dummy_heading = 1
ocn_off: true|object_number_off = 1
bullet: true|bullet = 1
table_header: true|table_header = 1
code_linenumbers: true|code_linenumbers = 1
lang|lang IS NOT NULL
indent|indent_base != 0 OR indent_hang != 0
"

# repeatable ssp property -> child table
LISTS="
image|object_images
stow_link|object_links
anchor_tag|object_anchors
lev4_subtoc|object_subtoc
"

report() { echo "  MISMATCH $1: .ssp $2, db $3"; }

# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the
# database beside it (pod/<uid>.ocda.db)
for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
  [ -f "$ssp" ] || continue
  base=$(basename "$ssp" .ssp)
  db="$OUT_DIR/pod/$base.ocda.db"
  if [ ! -f "$db" ]; then
    echo "MISSING db for $base"
    FAILURES=$((FAILURES + 1))
    continue
  fi
  DOCS=$((DOCS + 1))
  doc_fail=0

  # object population
  s_objs=$(grep -c '^\[' "$ssp" || true)
  d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR)
  if [ "$s_objs" != "$d_objs" ]; then
    echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1
  fi

  # per section
  for sec in head toc body endnotes glossary bibliography bookindex blurb; do
    s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp")
    d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR)
    if [ "$s_sec" != "$d_sec" ]; then
      [ $doc_fail -eq 0 ] && echo "$base"
      report "section $sec" "$s_sec" "$d_sec"; doc_fail=1
    fi
  done

  # single valued properties
  echo "$CHECKS" | while IFS='|' read -r prop pred; do
    [ -z "$prop" ] && continue
    case "$prop" in
      *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;;
      *)         s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;;
    esac
    d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR)
    if [ "$s_n" != "$d_n" ]; then
      echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n"
    fi
  done > "$OUT_DIR/.props.$$"
  if [ -s "$OUT_DIR/.props.$$" ]; then
    [ $doc_fail -eq 0 ] && echo "$base"
    sed 's/^[^:]*: /  /' "$OUT_DIR/.props.$$"
    doc_fail=1
  fi
  rm -f "$OUT_DIR/.props.$$"

  # repeatable properties against their tables
  echo "$LISTS" | while IFS='|' read -r prop table; do
    [ -z "$prop" ] && continue
    s_n=$(grep -c "^\.$prop: " "$ssp" || true)
    d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR)
    if [ "$s_n" != "$d_n" ]; then
      echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n"
    fi
  done > "$OUT_DIR/.lists.$$"
  if [ -s "$OUT_DIR/.lists.$$" ]; then
    [ $doc_fail -eq 0 ] && echo "$base"
    sed 's/^[^:]*: /  /' "$OUT_DIR/.lists.$$"
    doc_fail=1
  fi
  rm -f "$OUT_DIR/.lists.$$"

  # every image the abstraction referred to and found is carried in the file
  want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR)
  have=$(sqlite3 "$db" "select count(*) from files f where f.role='image'
           and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR)
  if [ "$want" != "$have" ]; then
    [ $doc_fail -eq 0 ] && echo "$base"
    report "images carried" "$want referenced" "$have stored"; doc_fail=1
  fi

  # the text itself, on the first and last citable object
  for pick in "min" "max"; do
    ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "")
    [ -z "$ocn" ] && continue
    sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true
    awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \
      "$ssp" > "$OUT_DIR/.ssptxt.$$"
    if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then
      [ $doc_fail -eq 0 ] && echo "$base"
      echo "  MISMATCH text of ocn $ocn"
      doc_fail=1
    fi
    rm -f "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$"
  done

  # the file says what it is
  sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '')
  if [ -z "$sv" ]; then
    [ $doc_fail -eq 0 ] && echo "$base"
    echo "  MISSING metadata schema.version"; doc_fail=1
  fi

  FAILURES=$((FAILURES + doc_fail))
done

rm -rf "$OUT_DIR"

if [ "$FAILURES" -eq 0 ]; then
  echo "PASS: .ssp and .ocda.db agree for all $DOCS documents"
  exit 0
else
  echo "FAIL: $FAILURES document(s) disagree"
  exit 1
fi