output: the abstraction artefacts live with the pod

pod/ holds all document source representations:
  pod/<doc>/                             source tree
  pod/<doc>/media/abstraction/<uid>.ssp  abstraction, as text
  pod/<doc>.zip                          tree, zipped, .ssp included
  pod/<doc>.digests.txt                  sha256s of what is in them
  pod/<uid>.ocda.db                      abstraction, as sqlite db

<lang>/abstraction/ is gone.
pod/<doc>/media/abstraction/<uid>.ssp preferred as having the
images (found within the pod tree) which .ssp needs to reproduce a
document but does not carry on its own.

The .ocda.db sits carries the images as well and (like the
pod.zip) can be used to reproduce a document directly.

digests.txt now covers the database as well as the zip, the source
and the .ssp; (as does the metadata html page).

Two ordering issues addressed:
- the pod builder clean-slates pod/<doc>/ before regenerating it,
  and the .ssp is now written before that runs. The clean slate
  now leaves media/abstraction/ alone, and the .ssp writer clears
  that directory itself on the first language of a run, so a .ssp
  for a language the document no longer is removed and cannot be
  bundled.
- for a multi-language document the .ssp files accumulate one
  language at a time and are bundled on the last, which is why the
  directory cannot simply be emptied by whichever (language) gets
  there first.

(assisted by Claude-Code)

5 files changed, 115 insertions, 33 deletions

diff --git a/org/out_metadata.org b/org/out_metadata.org
index 8606c69..71bfe9f 100644
--- a/org/out_metadata.org
+++ b/org/out_metadata.org
@@ -195,6 +195,23 @@ if (doc_matters.opt.action.html_link_markup_source) {
       writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n  ", fn_pod);
     }
   }
+  if (doc_matters.opt.action.show_abstraction_db) {
+    try { // sha digest and size for the abstraction database, beside the pod
+      string _db_fn = pths_pod.pod_dir_() ~ "/"
+        ~ doc_matters.src.doc_uid_out ~ ".ocda.db";
+      if (_db_fn.exists) {
+        auto data = (cast(byte[]) (_db_fn).read);
+        metadata_ ~= "<p class=\"lev2\">";
+        metadata_ ~= "<tt>" ~ data.sha256Of.toHexString ~ "::"
+          ~ data.length.to!string ~ "</tt> - "
+          ~ doc_matters.src.doc_uid_out ~ ".ocda.db";
+        metadata_ ~= "</p>";
+      }
+    } catch (Exception ex) {
+      writeln("WARNING, abstraction db not read for metadata page: ",
+        doc_matters.src.doc_uid_out, ".ocda.db");
+    }
+  }
 }
 if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) {
   metadata_ ~= "<hr /><p class=\"lev0\">Topics:</p>";
diff --git a/org/out_src_abstraction_ocda_peg_ssp.org b/org/out_src_abstraction_ocda_peg_ssp.org
index a541513..0edad7d 100644
--- a/org/out_src_abstraction_ocda_peg_ssp.org
+++ b/org/out_src_abstraction_ocda_peg_ssp.org
@@ -84,6 +84,7 @@ template sspObjectRecord() {
   import std.stdio;
   import std.string;
   import std.array;
+  import sisudoc.ocda.io_in.paths_source;
   mixin sspEscape;
   string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) {
     string[] output;
@@ -395,6 +396,7 @@ template spineAbstractionTxt() {
   import std.stdio;
   import std.string;
   import std.array;
+  import sisudoc.ocda.io_in.paths_source;
   mixin sspObjectRecord;
   /+ ↓ the whole .ssp for a document, as lines. the file writer below emits
        these, and the abstraction db is built by reading them back, so the
@@ -544,19 +546,34 @@ template spineAbstractionTxt() {
     }
     return output;
   }
-  void spineAbstractionTxt(D)(D doc) {
+  @trusted void spineAbstractionTxt(D)(D doc) {
     auto doc_matters = doc.matters;
     string[] output = sspDocumentLines(doc);
-    /+ ↓ write to file +/
-    /+ path: <output_path>/<language>/abstraction/<doc_uid_out>.ssp +/
-    string out_root = (doc_matters.output_path.length > 0)
-      ? doc_matters.output_path : "";
-    string base_pth = (out_root
-      .chainPath(doc_matters.src.language, "abstraction")
-      .asNormalizedPath).array;
+    /+ ↓ write to file, into the pod's own tree:
+         pod/<doc>/media/abstraction/<doc_uid_out>.ssp
+         .
+         where it sits beside pod/<doc>/media/image/, which is what a .ssp
+         needs to be a document source: the file describes its images by
+         name, size, pixels and digest, and they have to be findable. Written
+         here once; --pod2 then digests and zips it where it stands rather
+         than copying it in from an output directory of its own.
+    +/
+    auto pths_pod = spinePathsPods!()(doc_matters);
+    string base_pth
+      = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string;
     try {
       if (!exists(base_pth)) {
         base_pth.mkdirRecurse;
+      } else if (doc_matters.src.language
+        == doc_matters.pod.manifest_list_of_languages[0]
+      ) {
+        /+ ↓ this directory is the one part of the pod written before the pod
+             is built, so the pod builder leaves it alone. It therefore clears
+             itself here, on the first language of a run, or a .ssp for a
+             language the document no longer has would linger and be bundled +/
+        foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) {
+          _f.remove;
+        }
       }
     } catch (Exception ex) {
     }
diff --git a/org/out_src_abstraction_sqlite_db.org b/org/out_src_abstraction_sqlite_db.org
index 9e425bf..9144fdd 100644
--- a/org/out_src_abstraction_sqlite_db.org
+++ b/org/out_src_abstraction_sqlite_db.org
@@ -42,6 +42,7 @@ template spineAbstractionDb() {
   import std.array;
   import d2sqlite3;
   import sisudoc.outputs.io_out.paths_output;
+  import sisudoc.ocda.io_in.paths_source;
   /+ ↓ the abstraction is passed in rather than taken from doc, because the
        caller hands over one that has been through the .ssp: written as text
        and read back. so the database is a function of the .ssp and cannot
@@ -49,11 +50,16 @@ template spineAbstractionDb() {
   +/
   void spineAbstractionDb(D,A)(D doc, A doc_abstraction) {
     auto doc_matters     = doc.matters;
-
-    /+ ↓ determine output path +/
-    auto out_pth = spineOutPaths!()(doc_matters.output_path, doc_matters.src.language);
-    string base_dir = "abstraction";
-    string base_pth = ((out_pth.output_base.chainPath(base_dir)).asNormalizedPath).array;
+    /+ ↓ output path: beside the pod's zip and digests, not under
+         <lang>/abstraction/. the database carries its own images, so it is a
+         distributable artefact in its own right and belongs with the others:
+           pod/<doc>/            the source tree
+           pod/<doc>.zip         the same, zipped
+           pod/<doc>.digests.txt what is in them
+           pod/<uid>.ocda.db     the abstraction, one per language
+    +/
+    auto pths_pod = spinePathsPods!()(doc_matters);
+    string base_pth = pths_pod.pod_dir_();
     try {
       if (!exists(base_pth)) {
         base_pth.mkdirRecurse;
diff --git a/org/out_src_pod.org b/org/out_src_pod.org
index 793fa49..b987151 100644
--- a/org/out_src_pod.org
+++ b/org/out_src_pod.org
@@ -33,13 +33,35 @@ template spinePod() {
     <<source_pod_init>>
     if (doc_matters.opt.action.source_or_pod) {
       try {
-        /+ ↓ clean slate: remove per-document pod directory before regeneration,
-           but only on the first language of a multi-language document,
-           so that subsequent languages' files are not wiped +/
+        /+ ↓ clean slate: remove the per-document pod directory before
+           regeneration, but only on the first language of a multi-language
+           document, so that subsequent languages' files are not wiped.
+           .
+           media/abstraction/ is left alone: the .ssp files there are written
+           by the abstraction stage, which runs before this, and for a
+           multi-language document they accumulate one language at a time.
+           That directory clears itself, in the .ssp writer, on the first
+           language of a run.
+        +/
         if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0]) {
           string doc_pod_dir = pths_pod.base_filesystem_(doc_matters.src.filename);
+          string keep_dir
+            = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string;
           if (exists(doc_pod_dir) && doc_pod_dir.isDir) {
-            doc_pod_dir.rmdirRecurse;
+            foreach (string _e; dirEntries(doc_pod_dir, SpanMode.shallow)) {
+              if (_e.isDir) {
+                if (keep_dir.startsWith(_e)) {   // media/, which holds abstraction/
+                  foreach (string _m; dirEntries(_e, SpanMode.shallow)) {
+                    if (_m == keep_dir) { continue; }
+                    if (_m.isDir) { _m.rmdirRecurse; } else { _m.remove; }
+                  }
+                } else {
+                  _e.rmdirRecurse;
+                }
+              } else {
+                _e.remove;
+              }
+            }
           }
         }
         {
@@ -177,24 +199,24 @@ auto pod_zipMakeReady(M,P,S)(M doc_matters, P pths_pod, S _st) {
         }
       }
     }
-  } { // bundle abstraction .ssp file (only for --pod2)
+  } { // the abstraction .ssp, written in place by --show-abstraction (only for --pod2)
     if (doc_matters.opt.action.pod2) {
-      if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp generated
-        import sisudoc.outputs.io_out.paths_output;
-        /+ doc_uid_out for any language follows the same pattern, differing
+      if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp written
+        /+ the .ssp is written straight into pod/<doc>/media/abstraction/,
+           so there is nothing to copy here: it is digested and added to
+           the zip where it stands.
+           doc_uid_out for any language follows the same pattern, differing
            only in the trailing ".{lng}". Strip the current language to
-           reuse the base across all languages. +/
+           reuse the base across all languages.
+        +/
         string _doc_uid_base
           = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length];
         foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions
-          auto out_pth_lng = spineOutPaths!()(doc_matters.output_path, _lang);
-          string abstraction_dir = ((out_pth_lng.output_base.chainPath("abstraction")).asNormalizedPath).array;
           string ssp_filename = _doc_uid_base ~ _lang ~ ".ssp";
-          string fn_src_in = ((abstraction_dir.chainPath(ssp_filename)).asNormalizedPath).array.to!string;
           auto fn_src_out_pod_zip_base
             = pths_pod.abstraction_root(doc_matters.src.filename).zpod.to!string
             ~ "/" ~ ssp_filename;
-          auto fn_src_out_filesystem
+          auto fn_src_in
             = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string
             ~ "/" ~ ssp_filename;
           if (exists(fn_src_in)) {
@@ -204,13 +226,26 @@ auto pod_zipMakeReady(M,P,S)(M doc_matters, P pths_pod, S _st) {
               _digests[_lang]["ssp"] ~= file_.sha256hash.toHexString
                 ~ "::" ~ file_.fileSize.to!string ~ " - " ~ ssp_filename ~ "\n";
             }
-            fn_src_in.copy(fn_src_out_filesystem);
             zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
           } else {
             if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) {
               writeln("WARNING (io) src out NOT found (abstraction): ", fn_src_in);
             }
           }
+          /+ ↓ the .ocda.db sits beside the pod rather than inside it: it
+               carries the images itself, and putting it in the zip would
+               store them twice. digested here so that digests.txt covers
+               every artefact the pod directory holds for this document
+          +/
+          {
+            string db_filename = _doc_uid_base ~ _lang ~ ".ocda.db";
+            string fn_db = pths_pod.pod_dir_() ~ "/" ~ db_filename;
+            if (exists(fn_db)) {
+              ST_file_stat file_ = getFileStat(fn_db);
+              _digests[_lang]["ocda_db"] ~= file_.sha256hash.toHexString
+                ~ "::" ~ file_.fileSize.to!string ~ " - " ~ db_filename ~ "\n";
+            }
+          }
         }
       }
     }
@@ -578,7 +613,10 @@ void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) {
           f.writeln(_digests[_lang]["ssi"]);
         }
         if (("ssp" in _digests[_lang]) && (_digests[_lang]["ssp"].length > 0)) {
-          f.writeln(_digests[_lang]["ssp"]);
+          f.writeln(_digests[_lang]["ssp"].strip);
+        }
+        if (("ocda_db" in _digests[_lang]) && (_digests[_lang]["ocda_db"].length > 0)) {
+          f.writeln(_digests[_lang]["ocda_db"].strip);
         }
       }
     }
diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org
index ec8bbc9..4266e8c 100644
--- a/org/tests_for_document_abstraction_shell_scripts.org
+++ b/org/tests_for_document_abstraction_shell_scripts.org
@@ -427,10 +427,12 @@ lev4_subtoc|object_subtoc
 
 report() { echo "  MISMATCH $1: .ssp $2, db $3"; }
 
-for ssp in "$OUT_DIR"/*/abstraction/*.ssp; do
+# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the
+# database beside it (pod/<uid>.ocda.db)
+for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
   [ -f "$ssp" ] || continue
-  db="${ssp%.ssp}.ocda.db"
   base=$(basename "$ssp" .ssp)
+  db="$OUT_DIR/pod/$base.ocda.db"
   if [ ! -f "$db" ]; then
     echo "MISSING db for $base"
     FAILURES=$((FAILURES + 1))
@@ -695,10 +697,12 @@ $SPINE_BIN --show-abstraction --show-abstraction-db --skip-output \
 
 COUNT=0
 FAILURES=0
-for db in "$OUT_DIR"/*/abstraction/*.ocda.db; do
+# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp inside it
+# (pod/<doc>/media/abstraction/<uid>.ssp)
+for db in "$OUT_DIR"/pod/*.ocda.db; do
   [ -f "$db" ] || continue
-  ssp="${db%.ocda.db}.ssp"
-  base=$(basename "$ssp")
+  base=$(basename "$db" .ocda.db)
+  ssp=$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)
   COUNT=$((COUNT + 1))
   if [ ! -f "$ssp" ]; then
     echo "MISSING .ssp for $base"