pod zip fixesHEADmain

- serial processing (need to be built serially)
- multilingual pods, copy all languages before zip

3 files changed, 107 insertions, 113 deletions

diff --git a/src/sisudoc/io_out/hub.d b/src/sisudoc/io_out/hub.d
index b68eb0d..0b7fe8c 100644
--- a/src/sisudoc/io_out/hub.d
+++ b/src/sisudoc/io_out/hub.d
@@ -70,17 +70,11 @@ template outputHub() {
       auto msg = Msg!()(doc_matters);
       if (sched == outTask.source_or_pod) {
         msg.v("spine (doc reform) source processing... ");
-        if (doc_matters.opt.action.pod) {
-          msg.v("spine (doc reform) source pod processing... ");
-        }
+        if (doc_matters.opt.action.pod) { msg.v("spine (doc reform) source pod processing... "); }
         import sisudoc.io_out.source_pod;
         spinePod!()(doc_matters);
-        if (doc_matters.opt.action.source) {
-          msg.vv("spine (doc reform) source done");
-        }
-        if (doc_matters.opt.action.pod) {
-          msg.vv("spine (doc reform) source pod done");
-        }
+        if (doc_matters.opt.action.source) { msg.vv("spine (doc reform) source done"); }
+        if (doc_matters.opt.action.pod) { msg.vv("spine (doc reform) source pod done"); }
       }
       if (sched == outTask.epub) {
         msg.v("epub3 processing... ");
@@ -140,9 +134,7 @@ template outputHub() {
         msg.vv("sqlite done");
       }
     }
-    if (doc_matters.opt.action.vox_gt0) {
-      writeln(" ", doc_matters.src.filename_base);
-    }
+    if (doc_matters.opt.action.vox_gt0) { writeln(doc_matters.src.filename_base); }
     if (!(doc_matters.opt.action.parallelise_subprocesses)) {
       foreach(schedule; doc_matters.opt.action.output_task_scheduler) {
         Scheduled!()(schedule, doc_abstraction, doc_matters);
diff --git a/src/sisudoc/io_out/metadata.d b/src/sisudoc/io_out/metadata.d
index 95fc465..0917969 100644
--- a/src/sisudoc/io_out/metadata.d
+++ b/src/sisudoc/io_out/metadata.d
@@ -50,7 +50,7 @@
 module sisudoc.io_out.metadata;
 // @safe:
 template outputMetadata() {
-  void outputMetadata(T)(T  doc_matters) {
+  void outputMetadata(T)(T doc_matters) {
     string inline_search_form(M)(
       M  doc_matters,
     ) {
@@ -393,9 +393,7 @@ string theme_light_1 = format(q"┃
       } catch (ErrnoException ex) {
         // Handle error
       }
-      if (doc_matters.opt.action.vox_gt0) {
-        writeln(" ", pth_html.fn_scroll("metadata." ~ doc_matters.src.filename));
-      }
+      if (doc_matters.opt.action.vox_gt0) { writeln(" ", pth_html.fn_scroll("metadata." ~ doc_matters.src.filename)); }
     }
     static auto mkup = InlineMarkup();
     import sisudoc.io_out.html_snippet;
@@ -524,11 +522,6 @@ string theme_light_1 = format(q"┃
         }
       }
     }
-    metadata_ ~= "<p class=\"lev2\"><tt>"
-      ~ doc_matters.doc_digest.markup_doc.toHexString
-      ~ "</tt> - "
-      ~ doc_matters.src.filename
-      ~ "</p>";
     if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) {
       metadata_ ~= "<hr /><p class=\"lev0\">Topics:</p>";
       string[] _top = ["", "", "", "", ""];
diff --git a/src/sisudoc/io_out/source_pod.d b/src/sisudoc/io_out/source_pod.d
index 6b880d9..2b9953f 100644
--- a/src/sisudoc/io_out/source_pod.d
+++ b/src/sisudoc/io_out/source_pod.d
@@ -73,33 +73,38 @@ template spinePod() {
     auto lang = Lang();
     static auto rgx_files = RgxFiles();
     assert (doc_matters.src.filename.match(rgx_files.src_fn));
-    try {
-      {
-        pod_archive_directory_tree(doc_matters, pths_pod);
-      }
-      auto t = pod_zip_make_ready(doc_matters, pths_pod);
-      static assert(t.length==3);
-      auto zip = t[0];
-      auto fn_pod = t[1];
-      auto _digests = t[2];
-      {
-        zipArchive(doc_matters, fn_pod, zip);
-      } {
-        zipArchiveDigest(doc_matters, fn_pod, _digests);
+    if (doc_matters.opt.action.pod) {
+      try {
+        {
+          podArchive_directory_tree(doc_matters, pths_pod);
+        }
+        {
+          struct STsrcDigests {
+            std.zip.ZipArchive     zip;
+            string                 fn_pod;
+            string[string][string] digests;
+          }
+          STsrcDigests _st;
+          _st = pod_zipMakeReady(doc_matters, pths_pod, _st);
+          {
+            zipArchive(doc_matters, _st.fn_pod, _st.zip);
+            if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) {
+              zipArchiveDigest(doc_matters, _st.fn_pod, _st.digests);
+            }
+          }
+        }
+      } catch (ErrnoException ex) {
+        // Handle error
       }
-    } catch (ErrnoException ex) {
-      // Handle error
     }
   }
-  void pod_archive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure
+  void podArchive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure
     if (!exists(pths_pod.pod_dir_())) {
       // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
       pths_pod.pod_dir_().mkdirRecurse;
     }
     if (doc_matters.opt.action.source_or_pod) {
-      if (doc_matters.opt.action.vox_gt0) {
-        writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod);
-      }
+      // if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); }
       if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) {
         pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
       }
@@ -132,10 +137,9 @@ template spinePod() {
       ));
     }
   }
-  auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
+  auto pod_zipMakeReady(M,P,S)(M doc_matters, P pths_pod, S _st) {
     auto pth_dr_doc_src = doc_matters.src_path_info;
-    if (doc_matters.opt.action.debug_do_pod
-    && doc_matters.opt.action.vox_gt1) {
+    if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
       writeln(__LINE__, ": ",
         doc_matters.src.filename, " -> ",
         pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod
@@ -170,11 +174,10 @@ template spinePod() {
             fn_src_in.copy(fn_src_out_filesystem);
           }
           if (doc_matters.opt.action.pod) {
-            zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip);
+            zip = podArchive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip);
           }
         } else {
-          if (doc_matters.opt.action.debug_do_pod
-          && doc_matters.opt.action.vox_gt1) {
+          if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
             writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
           }
         }
@@ -195,11 +198,10 @@ template spinePod() {
           fn_src_in.copy(fn_src_out_filesystem);
         }
         if (doc_matters.opt.action.pod) {
-          zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
+          zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
         }
       } else {
-        if (doc_matters.opt.action.debug_do_pod
-        && doc_matters.opt.action.vox_gt1) {
+        if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
           writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
         }
       }
@@ -223,8 +225,7 @@ template spinePod() {
             = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
           Node _pmy;
           string _pm = "doc:\n  filename: " ~ doc_matters.src.filename ~ "\n  language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n";
-          if (doc_matters.opt.action.debug_do_pod
-          && doc_matters.opt.action.vox_gt1) {
+          if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
             try {
               _pmy = Loader.fromString(_pm).load();
             } catch (ErrnoException ex) {
@@ -242,7 +243,7 @@ template spinePod() {
             pod_filelist_yaml_string.writeln(_pm);
           }
           if (doc_matters.opt.action.pod) {
-            zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip);
+            zip = podArchive("string", _pm, fn_src_out_pod_zip_base, zip);
           }
         }
       }
@@ -256,47 +257,55 @@ template spinePod() {
         = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path:
       string[] filelist_src_out_pod_arr;
       string[] filelist_src_zpod_arr;
-      if (exists(fn_src_in)) { // what of language?
-        debug(io) { writeln("(io debug) src in found: ", fn_src_in); }
-        { // take DIGEST write to pod file digests.txt
-          auto data = (cast(byte[]) (fn_src_in).read);
-          _digests["en"]["sst"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename; // FIX language issue
-          // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename);
-        }
-        filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
-        filelist_src_zpod_arr ~= fn_src_out_inside_pod;
-        string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename;
-        if (doc_matters.opt.action.source_or_pod) {
-          fn_src_in.copy(fn_src_out_filesystem);
-        }
-        if (doc_matters.opt.action.pod) {
-          auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])");
-          if (auto _x = fn_src_in.match(_rgx)){
-            if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
-              string _path_to_pod = _x.captures["path_to_pod"];
-              string _podname = _x.captures["podname"];
-              string _root_to_lang = _x.captures["from_root"];
-              string _language = _x.captures["language"];
-              string _filename = _x.captures["filename"];
-              foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
-                string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
-                string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
-                zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
+      if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssm parsed
+        foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions
+          string fn_src_out_filesystem_lng
+            = pths_pod.fn_doc(doc_matters.src.filename, _lang).filesystem_open_zpod.to!string;
+          string _sstm = (doc_matters.pod.manifest_path ~ "/media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename);
+          // writeln(_sstm);
+          if (exists(_sstm)) { // what of language?
+            debug(io) { writeln("(io debug) src in found: ", _sstm); }
+            { // take DIGEST write to pod file digests.txt
+              auto data = (cast(byte[]) (_sstm).read);
+              _digests[_lang]["sstm"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename ~ " - [" ~ _lang ~ "]";
+              // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename);
+            }
+            filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
+            filelist_src_zpod_arr ~= fn_src_out_inside_pod;
+            string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename;
+            if (doc_matters.opt.action.source_or_pod) {
+              _sstm.copy(fn_src_out_filesystem_lng);
+            }
+            if (doc_matters.opt.action.pod) {
+              auto _rgx_sstm = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])");
+              if (auto _x = _sstm.match(_rgx_sstm)){
+                if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { // again wait until all language versions of .ssm parsed
+                  string _path_to_pod = _x.captures["path_to_pod"];
+                  string _podname = _x.captures["podname"];
+                  string _root_to_lang = _x.captures["from_root"];
+                  string _language = _x.captures["language"]; // .ssi inserts expected to have same name across languages
+                  string _filename = _x.captures["filename"];
+                  foreach (_lang1; doc_matters.pod.manifest_list_of_languages) { // do for all language versions
+                    string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename;
+                    string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename;
+                    // writeln("\nin:  ", _pth_mkup_src_in, "\nout: ", _pth_mkup_src_out); // DEBUG, REMOVE
+                    zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
+                  }
+                }
+              } else {
+                zip = podArchive("file_path_text", _sstm, fn_src_out_pod_zip_base, zip);
               }
             }
           } else {
-            zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
+            if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
+              writeln("WARNING (io) src in NOT found (markup source): ", _sstm);
+            }
           }
         }
-      } else {
-        if (doc_matters.opt.action.debug_do_pod
-        && doc_matters.opt.action.vox_gt1) {
-          writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
-        }
       }
     } { // bundle insert files (.ssi) - get digest
       if (doc_matters.srcs.file_insert_list.length > 0) {
-        auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])");
+        auto _rgx_ssi = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])");
         foreach (insert_file; doc_matters.srcs.file_insert_list) {
           debug(pod) {
             writeln(
@@ -308,7 +317,7 @@ template spinePod() {
               ).zpod
             );
           }
-          if (auto _x = insert_file.match(_rgx)){
+          if (auto _x = insert_file.match(_rgx_ssi)){
             if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
               string _path_to_pod = _x.captures["path_to_pod"];
               string _podname = _x.captures["podname"];
@@ -334,11 +343,10 @@ template spinePod() {
                     _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere
                   }
                   if (doc_matters.opt.action.pod) {
-                     zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
+                     zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
                   }
                 } else {
-                  if (doc_matters.opt.action.debug_do_pod
-                  && doc_matters.opt.action.vox_gt1) {
+                  if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
                     writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in);
                   }
                 }
@@ -369,11 +377,10 @@ template spinePod() {
                 fn_src_in.copy(fn_src_out_filesystem);
               }
               if (doc_matters.opt.action.pod) {
-                zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
+                zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
               }
             } else {
-              if (doc_matters.opt.action.debug_do_pod
-              && doc_matters.opt.action.vox_gt1) {
+              if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
                 writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
               }
             }
@@ -381,10 +388,14 @@ template spinePod() {
         }
       }
     }
-    auto t = tuple(zip, fn_pod, _digests);
-    return t;
+    {
+      _st.zip     = zip;
+      _st.fn_pod  = fn_pod;
+      _st.digests = _digests;
+    }
+    return _st;
   }
-  @system auto pod_archive(Z)(
+  @system auto podArchive(Z)(
     string _source_type,
     string _data_in,
     string _pth_out,
@@ -425,6 +436,7 @@ template spinePod() {
   void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) {
     import sisudoc.io_out.paths_output;
     auto pths_pod = spinePathsPods!()(doc_matters);
+    char[] _zip_digest;
     try {
       if (!exists(pths_pod.pod_dir_())) {
         // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
@@ -434,23 +446,20 @@ template spinePod() {
       // Handle error
     }
     try {
-      writeln(pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt");
+      // if (doc_matters.opt.action.vox_gt1) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); }
       string _digest_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt";
-      writeln(_digest_fn);
+      // if (doc_matters.opt.action.vox_gt1) { writeln(_digest_fn); }
       auto f = File(_digest_fn, "w");
       if (exists(fn_pod)) {
         try {
-          if (doc_matters.opt.action.vox_gt0
-          && doc_matters.opt.action.pod) {
-            auto data = (cast(byte[]) (fn_pod).read);
-            if (doc_matters.opt.action.vox_gt1) {
-              writeln(doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip");
-            }
-            if (doc_matters.opt.action.pod) {
-              auto _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip");
-              writeln(_zip_digest);
-              f.writeln(_zip_digest);
-            }
+          auto data = (cast(byte[]) (fn_pod).read);
+          // if (doc_matters.opt.action.vox_gt1) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); }
+          if (doc_matters.opt.action.pod) {
+            _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip");
+            if (doc_matters.opt.action.vox_gt0) { writeln(" ", _zip_digest); }
+            if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, "/"); }
+            if (doc_matters.opt.action.vox_gt0) { writeln(" ", _digest_fn); }
+            f.writeln(_zip_digest);
           }
         } catch (ErrnoException ex) {
           // Handle errors
@@ -458,19 +467,19 @@ template spinePod() {
       }
       foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
         if (_lang in _digests) {
-          if (("sst" in _digests[_lang]) && (_digests[_lang]["sst"].length > 0)) {
-            writeln(_digests[_lang]["sst"]);
-            f.writeln(_digests[_lang]["sst"]);
+          if (("sstm" in _digests[_lang]) && (_digests[_lang]["sstm"].length > 0)) {
+            // if (doc_matters.opt.action.vox_gt1) { writeln(_digests[_lang]["sstm"]); }
+            f.writeln(_digests[_lang]["sstm"]);
           }
           if (("ssi" in _digests[_lang]) && (_digests[_lang]["ssi"].length > 0)) {
-            writeln(_digests[_lang]["ssi"]);
+            // if (doc_matters.opt.action.vox_gt1) { writeln(_digests[_lang]["ssi"]); }
             f.writeln(_digests[_lang]["ssi"]);
           }
         }
       }
       if ("shared" in _digests) {
         if (("images" in _digests["shared"]) && (_digests["shared"]["images"].length > 0)) {
-          writeln(_digests["shared"]["images"]);
+          // if (doc_matters.opt.action.vox_gt1) { writeln(_digests["shared"]["images"]); }
           f.writeln(_digests["shared"]["images"]);
         }
       }