output: the abstraction artefacts live with the pod

pod/ holds all document source representations:
  pod/<doc>/                             source tree
  pod/<doc>/media/abstraction/<uid>.ssp  abstraction, as text
  pod/<doc>.zip                          tree, zipped, .ssp included
  pod/<doc>.digests.txt                  sha256s of what is in them
  pod/<uid>.ocda.db                      abstraction, as sqlite db

<lang>/abstraction/ is gone.
pod/<doc>/media/abstraction/<uid>.ssp preferred as having the
images (found within the pod tree) which .ssp needs to reproduce a
document but does not carry on its own.

The .ocda.db sits carries the images as well and (like the
pod.zip) can be used to reproduce a document directly.

digests.txt now covers the database as well as the zip, the source
and the .ssp; (as does the metadata html page).

Two ordering issues addressed:
- the pod builder clean-slates pod/<doc>/ before regenerating it,
  and the .ssp is now written before that runs. The clean slate
  now leaves media/abstraction/ alone, and the .ssp writer clears
  that directory itself on the first language of a run, so a .ssp
  for a language the document no longer is removed and cannot be
  bundled.
- for a multi-language document the .ssp files accumulate one
  language at a time and are bundled on the last, which is why the
  directory cannot simply be emptied by whichever (language) gets
  there first.

(assisted by Claude-Code)

4 files changed, 106 insertions, 28 deletions

diff --git a/src/sisudoc/ocda/abstraction/ssp.d b/src/sisudoc/ocda/abstraction/ssp.d
index 3e80956..2501add 100644
--- a/src/sisudoc/ocda/abstraction/ssp.d
+++ b/src/sisudoc/ocda/abstraction/ssp.d
@@ -102,6 +102,7 @@ template sspObjectRecord() {
   import std.stdio;
   import std.string;
   import std.array;
+  import sisudoc.ocda.io_in.paths_source;
   mixin sspEscape;
   string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) {
     string[] output;
@@ -413,6 +414,7 @@ template spineAbstractionTxt() {
   import std.stdio;
   import std.string;
   import std.array;
+  import sisudoc.ocda.io_in.paths_source;
   mixin sspObjectRecord;
   /+ ↓ the whole .ssp for a document, as lines. the file writer below emits
        these, and the abstraction db is built by reading them back, so the
@@ -562,19 +564,34 @@ template spineAbstractionTxt() {
     }
     return output;
   }
-  void spineAbstractionTxt(D)(D doc) {
+  @trusted void spineAbstractionTxt(D)(D doc) {
     auto doc_matters = doc.matters;
     string[] output = sspDocumentLines(doc);
-    /+ ↓ write to file +/
-    /+ path: <output_path>/<language>/abstraction/<doc_uid_out>.ssp +/
-    string out_root = (doc_matters.output_path.length > 0)
-      ? doc_matters.output_path : "";
-    string base_pth = (out_root
-      .chainPath(doc_matters.src.language, "abstraction")
-      .asNormalizedPath).array;
+    /+ ↓ write to file, into the pod's own tree:
+         pod/<doc>/media/abstraction/<doc_uid_out>.ssp
+         .
+         where it sits beside pod/<doc>/media/image/, which is what a .ssp
+         needs to be a document source: the file describes its images by
+         name, size, pixels and digest, and they have to be findable. Written
+         here once; --pod2 then digests and zips it where it stands rather
+         than copying it in from an output directory of its own.
+    +/
+    auto pths_pod = spinePathsPods!()(doc_matters);
+    string base_pth
+      = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string;
     try {
       if (!exists(base_pth)) {
         base_pth.mkdirRecurse;
+      } else if (doc_matters.src.language
+        == doc_matters.pod.manifest_list_of_languages[0]
+      ) {
+        /+ ↓ this directory is the one part of the pod written before the pod
+             is built, so the pod builder leaves it alone. It therefore clears
+             itself here, on the first language of a run, or a .ssp for a
+             language the document no longer has would linger and be bundled +/
+        foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) {
+          _f.remove;
+        }
       }
     } catch (Exception ex) {
     }
diff --git a/src/sisudoc/outputs/io_out/create_abstraction_db.d b/src/sisudoc/outputs/io_out/create_abstraction_db.d
index e862aba..9b1af4d 100644
--- a/src/sisudoc/outputs/io_out/create_abstraction_db.d
+++ b/src/sisudoc/outputs/io_out/create_abstraction_db.d
@@ -60,6 +60,7 @@ template spineAbstractionDb() {
   import std.array;
   import d2sqlite3;
   import sisudoc.outputs.io_out.paths_output;
+  import sisudoc.ocda.io_in.paths_source;
   /+ ↓ the abstraction is passed in rather than taken from doc, because the
        caller hands over one that has been through the .ssp: written as text
        and read back. so the database is a function of the .ssp and cannot
@@ -67,11 +68,16 @@ template spineAbstractionDb() {
   +/
   void spineAbstractionDb(D,A)(D doc, A doc_abstraction) {
     auto doc_matters     = doc.matters;
-
-    /+ ↓ determine output path +/
-    auto out_pth = spineOutPaths!()(doc_matters.output_path, doc_matters.src.language);
-    string base_dir = "abstraction";
-    string base_pth = ((out_pth.output_base.chainPath(base_dir)).asNormalizedPath).array;
+    /+ ↓ output path: beside the pod's zip and digests, not under
+         <lang>/abstraction/. the database carries its own images, so it is a
+         distributable artefact in its own right and belongs with the others:
+           pod/<doc>/            the source tree
+           pod/<doc>.zip         the same, zipped
+           pod/<doc>.digests.txt what is in them
+           pod/<uid>.ocda.db     the abstraction, one per language
+    +/
+    auto pths_pod = spinePathsPods!()(doc_matters);
+    string base_pth = pths_pod.pod_dir_();
     try {
       if (!exists(base_pth)) {
         base_pth.mkdirRecurse;
diff --git a/src/sisudoc/outputs/io_out/metadata.d b/src/sisudoc/outputs/io_out/metadata.d
index ff9c72f..f2aaa53 100644
--- a/src/sisudoc/outputs/io_out/metadata.d
+++ b/src/sisudoc/outputs/io_out/metadata.d
@@ -529,6 +529,23 @@ string theme_light_1 = format(q"┃
           writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n  ", fn_pod);
         }
       }
+      if (doc_matters.opt.action.show_abstraction_db) {
+        try { // sha digest and size for the abstraction database, beside the pod
+          string _db_fn = pths_pod.pod_dir_() ~ "/"
+            ~ doc_matters.src.doc_uid_out ~ ".ocda.db";
+          if (_db_fn.exists) {
+            auto data = (cast(byte[]) (_db_fn).read);
+            metadata_ ~= "<p class=\"lev2\">";
+            metadata_ ~= "<tt>" ~ data.sha256Of.toHexString ~ "::"
+              ~ data.length.to!string ~ "</tt> - "
+              ~ doc_matters.src.doc_uid_out ~ ".ocda.db";
+            metadata_ ~= "</p>";
+          }
+        } catch (Exception ex) {
+          writeln("WARNING, abstraction db not read for metadata page: ",
+            doc_matters.src.doc_uid_out, ".ocda.db");
+        }
+      }
     }
     if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) {
       metadata_ ~= "<hr /><p class=\"lev0\">Topics:</p>";
diff --git a/src/sisudoc/outputs/io_out/source_pod.d b/src/sisudoc/outputs/io_out/source_pod.d
index b32abcd..0ed26db 100644
--- a/src/sisudoc/outputs/io_out/source_pod.d
+++ b/src/sisudoc/outputs/io_out/source_pod.d
@@ -73,13 +73,35 @@ template spinePod() {
     assert (doc_matters.src.filename.match(rgx_files.src_fn));
     if (doc_matters.opt.action.source_or_pod) {
       try {
-        /+ ↓ clean slate: remove per-document pod directory before regeneration,
-           but only on the first language of a multi-language document,
-           so that subsequent languages' files are not wiped +/
+        /+ ↓ clean slate: remove the per-document pod directory before
+           regeneration, but only on the first language of a multi-language
+           document, so that subsequent languages' files are not wiped.
+           .
+           media/abstraction/ is left alone: the .ssp files there are written
+           by the abstraction stage, which runs before this, and for a
+           multi-language document they accumulate one language at a time.
+           That directory clears itself, in the .ssp writer, on the first
+           language of a run.
+        +/
         if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0]) {
           string doc_pod_dir = pths_pod.base_filesystem_(doc_matters.src.filename);
+          string keep_dir
+            = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string;
           if (exists(doc_pod_dir) && doc_pod_dir.isDir) {
-            doc_pod_dir.rmdirRecurse;
+            foreach (string _e; dirEntries(doc_pod_dir, SpanMode.shallow)) {
+              if (_e.isDir) {
+                if (keep_dir.startsWith(_e)) {   // media/, which holds abstraction/
+                  foreach (string _m; dirEntries(_e, SpanMode.shallow)) {
+                    if (_m == keep_dir) { continue; }
+                    if (_m.isDir) { _m.rmdirRecurse; } else { _m.remove; }
+                  }
+                } else {
+                  _e.rmdirRecurse;
+                }
+              } else {
+                _e.remove;
+              }
+            }
           }
         }
         {
@@ -208,24 +230,24 @@ template spinePod() {
           }
         }
       }
-    } { // bundle abstraction .ssp file (only for --pod2)
+    } { // the abstraction .ssp, written in place by --show-abstraction (only for --pod2)
       if (doc_matters.opt.action.pod2) {
-        if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp generated
-          import sisudoc.outputs.io_out.paths_output;
-          /+ doc_uid_out for any language follows the same pattern, differing
+        if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp written
+          /+ the .ssp is written straight into pod/<doc>/media/abstraction/,
+             so there is nothing to copy here: it is digested and added to
+             the zip where it stands.
+             doc_uid_out for any language follows the same pattern, differing
              only in the trailing ".{lng}". Strip the current language to
-             reuse the base across all languages. +/
+             reuse the base across all languages.
+          +/
           string _doc_uid_base
             = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length];
           foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions
-            auto out_pth_lng = spineOutPaths!()(doc_matters.output_path, _lang);
-            string abstraction_dir = ((out_pth_lng.output_base.chainPath("abstraction")).asNormalizedPath).array;
             string ssp_filename = _doc_uid_base ~ _lang ~ ".ssp";
-            string fn_src_in = ((abstraction_dir.chainPath(ssp_filename)).asNormalizedPath).array.to!string;
             auto fn_src_out_pod_zip_base
               = pths_pod.abstraction_root(doc_matters.src.filename).zpod.to!string
               ~ "/" ~ ssp_filename;
-            auto fn_src_out_filesystem
+            auto fn_src_in
               = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string
               ~ "/" ~ ssp_filename;
             if (exists(fn_src_in)) {
@@ -235,13 +257,26 @@ template spinePod() {
                 _digests[_lang]["ssp"] ~= file_.sha256hash.toHexString
                   ~ "::" ~ file_.fileSize.to!string ~ " - " ~ ssp_filename ~ "\n";
               }
-              fn_src_in.copy(fn_src_out_filesystem);
               zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
             } else {
               if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) {
                 writeln("WARNING (io) src out NOT found (abstraction): ", fn_src_in);
               }
             }
+            /+ ↓ the .ocda.db sits beside the pod rather than inside it: it
+                 carries the images itself, and putting it in the zip would
+                 store them twice. digested here so that digests.txt covers
+                 every artefact the pod directory holds for this document
+            +/
+            {
+              string db_filename = _doc_uid_base ~ _lang ~ ".ocda.db";
+              string fn_db = pths_pod.pod_dir_() ~ "/" ~ db_filename;
+              if (exists(fn_db)) {
+                ST_file_stat file_ = getFileStat(fn_db);
+                _digests[_lang]["ocda_db"] ~= file_.sha256hash.toHexString
+                  ~ "::" ~ file_.fileSize.to!string ~ " - " ~ db_filename ~ "\n";
+              }
+            }
           }
         }
       }
@@ -539,7 +574,10 @@ template spinePod() {
             f.writeln(_digests[_lang]["ssi"]);
           }
           if (("ssp" in _digests[_lang]) && (_digests[_lang]["ssp"].length > 0)) {
-            f.writeln(_digests[_lang]["ssp"]);
+            f.writeln(_digests[_lang]["ssp"].strip);
+          }
+          if (("ocda_db" in _digests[_lang]) && (_digests[_lang]["ocda_db"].length > 0)) {
+            f.writeln(_digests[_lang]["ocda_db"].strip);
           }
         }
       }