aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-11 21:45:24 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-12 12:39:06 -0400
commitc6a479b9bd8bdd34ee9699e10247ebdeb183e032 (patch)
tree1eab495f7fbf5467c023651aa7f0ee5009215064
parentspine: a .ocda.db can be fetched, as a pod zip can (diff)
org renames
-rw-r--r--org/in_abstraction_artefacts.org (renamed from org/in_source_files.org)1564
-rw-r--r--org/in_markup_source_files.org801
-rw-r--r--org/in_source_artefact.org392
-rw-r--r--org/ocda_get.org284
-rw-r--r--org/out_ocda_peg_ssp.org (renamed from org/out_src_abstraction_ocda_peg_ssp.org)0
-rw-r--r--org/out_ocda_sqlite_db.org (renamed from org/out_src_abstraction_sqlite_db.org)0
6 files changed, 1493 insertions, 1548 deletions
diff --git a/org/in_source_files.org b/org/in_abstraction_artefacts.org
index 78141b7..d8ce545 100644
--- a/org/in_source_files.org
+++ b/org/in_abstraction_artefacts.org
@@ -17,768 +17,180 @@
- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
-* A. get _config file_, read in (.readText) [#A]
-** _module template_ :module:config_files:
+* A. load abstraction - sisudoc.ocda.abstraction.load
-#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d"
+#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
-/++
- read configuration files<BR>
- - read config files<BR>
- meta_config_files.d
-+/
-module sisudoc.ocda.io_in.read_config_files;
+module sisudoc.ocda.abstraction.load;
@safe:
-import std.file;
-import std.path;
-import sisudoc.ocda.meta;
-import sisudoc.ocda.io_in.paths_source;
-import sisudoc.ocda.meta.rgx_files;
-import sisudoc.ocda.meta.rgx;
-<<meta_config_file_hub_read_site_config>>
-<<meta_config_file_hub_read_document_config>>
-<<meta_config_file_hub_read_site_yaml>>
-#+END_SRC
-
-*** read config files (config local site & dr document make) (yaml)
-**** site configuration
-
-SEE NOTES on configuration hierarchy in spine.org
-
-#+NAME: meta_config_file_hub_read_site_config
-#+BEGIN_SRC d
-template readConfigSite() {
- @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) {
- mixin spineRgxIn;
- static auto rgx = RgxI();
- string conf_filename = "NONE";
- string config_file_str;
- string default_config_file_str = format(q"┃
-flag:
- act0: "--html"
- act1: "--html --epub"
-output:
- path: "%s"
-default:
- language: "en"
- papersize: "a4"
- text_wrap: "80"
- digest: "sha256"
-webserv:
- http: "%s"
- host: "%s"
- data_http: "%s"
- data_host: "%s"
- data_root_url: "%s"
- data_root_path: "%s"
- data_root_part: ""
- images_root_part: "image"
- cgi_search_form_title: "%s"
- cgi_http: "%s"
- cgi_host: "%s"
- cgi_bin_url: "%s"
- cgi_bin_subpath: "%s"
- cgi_bin_path: "%s"
- cgi_search_script: "%s"
- cgi_port: ""
- cgi_user: ""
- cgi_action: "%s"
- db_sqlite_path: "%s"
- db_sqlite_filename: "%s"
- db_pg_table: ""
- db_pg_user: ""
-┃",
- _cfg.processing_path_doc_root, // doc root
- _cfg.http_request_type, // http
- _cfg.http_host, // host / domain
- _cfg.http_request_type, // data "http" or "https"
- _cfg.http_host, // data domain "localhost"
- _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org"
- _cfg.processing_path_doc_root, // data root path
- _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search"
- _cfg.http_request_type, // cgi http
- _cfg.http_host, // cgi host
- _cfg.cgi_url_root, // cgi bin url
- _cfg.cgi_bin_subpath, // cgi bin path
- _cfg.cgi_bin_root, // cgi bin path
- _cfg.cgi_filename, // cgi filename
- _cfg.cgi_url_action, // cgi action
- _cfg.db_sqlite_path, // sqlite db path
- _cfg.db_sqlite_filename, // sqlite db filename
-);
- foreach(conf_fn; [_conf_file_details.config_filename_site]) {
- foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) {
- char[] conf_file;
- conf_filename = conf_fn;
- if (exists(pth)) {
- auto f_attrib = pth.getLinkAttributes;
- if (
- _conf_file_details.possible_config_path_locations.config_local_site.length == 1
- && f_attrib.attrIsFile
- ) {
- conf_file = pth.to!(char[]);
- conf_filename = pth.baseName;
- } else if (f_attrib.attrIsDir) {
- conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
- conf_filename = conf_fn;
- }
- try {
- if (exists(conf_file)) {
- if (conf_file.getLinkAttributes.attrIsFile) {
- if (_opt_action.vox_gt_2 || _opt_action.debug_do) {
- writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)");
- }
- config_file_str = conf_file.readText;
- break;
- }
- }
- } catch (ErrnoException ex) {
- } catch (FileException ex) {
- }
- }
- }
- if (config_file_str.length > 0) { break; }
- }
- if (config_file_str.length > 0) {
- import dyaml;
- Node yaml_root;
- try {
- yaml_root = Loader.fromString(config_file_str).load();
- } catch (Throwable) {
- import std.stdio;
- writeln("ERROR failed to read config file content, not parsed as yaml, program default used");
- conf_filename = "VIRTUAL";
- config_file_str = default_config_file_str;
- }
- }
- if (config_file_str.length == 0) { /+ use dummy default config file +/
- // writeln("WARNING config file NOT found, default provided");
- conf_filename = "VIRTUAL";
- config_file_str = default_config_file_str;
- }
- struct _ConfContent {
- string filename() {
- return conf_filename;
- }
- string filetype() {
- string _ft = "";
- if (content.match(rgx.yaml_config)) {
- _ft = "yaml";
- }
- return _ft;
- }
- string content() {
- return config_file_str;
- }
- }
- return _ConfContent();
- }
-}
-#+END_SRC
-
-**** document make/config
+/+ ↓ one way in, whatever the document is being read from
+ spine's pipeline is markup -> abstraction -> output. once the abstraction
+ is serialised, there is more than one thing an abstraction can be read
+ from, and a consumer should not have to know which it was handed:
+ .
+ .sst / .ssm + images the markup source
+ pod (dir) + images the same, bundled
+ pod .zip the same, zipped
+ .ssp + images the abstraction, as text
+ .ocda.db the abstraction, sqlite, images inside
+ .
+ A source may also be remote. A URL ending in .zip or in .ocda.db is
+ downloaded to a temp file (downloadSourceUrl in
+ ocda/io_in/read_zip_pod.d, guarded by rgx_url_source and by
+ --allow-downloads) and the local path is processed in its place.
-#+NAME: meta_config_file_hub_read_document_config
-#+HEADER: :noweb yes
-#+BEGIN_SRC d
-static template readConfigDoc() {
+ Those two travel because each is self-sufficient: a pod holds its
+ source and images, a .ocda.db holds its abstraction and images. A .ssp
+ does not, describing its images by name and digest without carrying
+ them, so it would arrive without them and is not fetched.
+ .
+ This module names those sources, tells them apart, and loads the two
+ that are self-describing artefacts, returning the value the parser
+ produces.
+ .
+ The three source forms are deliberately *not* loaded here. They are the
+ parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs
+ the environment, the options and the configuration that spine.d
+ assembles, none of which belongs in a loader. What this module gives
+ that case is the dispatch and a plain statement of where to go.
++/
+template spineAbstractionLoad() {
+ import std.algorithm : endsWith;
+ import std.conv : to;
import std.file;
import std.path;
- import sisudoc.ocda.meta;
- import sisudoc.ocda.io_in.paths_source;
- import sisudoc.ocda.meta.rgx_files;
- import sisudoc.ocda.meta.rgx;
- @system final auto readConfigDoc(M,E)(M _manifested, E _env) {
- mixin spineRgxIn;
- static auto rgx = RgxI();
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- string config_file_str;
- string conf_filename = "NONE";
- auto _conf_file_details = configFilePaths!()(_manifested, _env);
- string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make;
- foreach(conf_fn; [_conf_file_details.config_filename_document]) {
- foreach(pth; possible_config_path_locations) {
- char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
- conf_filename = conf_fn;
- if (config_file_str.length > 0) {
- break;
- }
- try {
- if (exists(conf_file)) {
- if (conf_file.getLinkAttributes.attrIsFile) {
- config_file_str = conf_file.readText;
- break;
- }
- }
- } catch (ErrnoException ex) {
- } catch (FileException ex) {
- }
- }
- if (config_file_str.length > 0) { break; }
- }
- struct _ConfContent {
- string filename() {
- return conf_filename;
- }
- string content() {
- return config_file_str;
- }
- string filetype() {
- string _ft = "";
- if (content.match(rgx.yaml_config)) {
- _ft = "yaml";
- }
- return _ft;
- }
- }
- return _ConfContent();
- }
-}
-#+END_SRC
-
-*** YAML config (config local site & dr document make) :file:config:hub:
+ import std.stdio;
+ import std.string;
+ import sisudoc.ocda.abstraction.db_in;
+ mixin spineAbstractionDbRead; // brings the .ssp reader with it
-#+NAME: meta_config_file_hub_read_site_yaml
-#+HEADER: :noweb yes
-#+BEGIN_SRC d
-static template configReadSiteYAML() {
- import std.file;
- import std.path;
- import sisudoc.ocda.meta;
- import sisudoc.ocda.io_in.paths_source;
- import sisudoc.ocda.meta.rgx_files;
- import sisudoc.ocda.meta.rgx;
- final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) {
- string _configuration = configReadInSiteYAML!()(_manifested, _env);
- auto _conf_file_details = configFilePaths!()(_manifested, _env);
- string _conf_yaml_fn = _conf_file_details.config_filename_site;
- YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
- return _yaml_conf;
- }
-}
-static template configReadDocYAML() {
- import std.file;
- import std.path;
- import sisudoc.ocda.meta;
- import sisudoc.ocda.io_in.paths_source;
- final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) {
- string _configuration = configReadInDocYAML!()(_manifested, _env);
- auto _conf_file_details = configFilePaths!()(_manifested, _env);
- string _conf_yaml_fn = _conf_file_details.config_filename_document;
- YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
- return _yaml_conf;
+ enum AbstractionSource {
+ unknown,
+ markup, // .sst or .ssm, with its images beside it
+ pod, // a directory holding pod.manifest
+ pod_zip, // that directory, zipped
+ ssp, // .ssp, the abstraction as text, images beside it
+ ocda_db, // .ocda.db, the abstraction as sqlite, images inside it
}
-}
-#+END_SRC
-* B. get _markup source_, read file (.readText) [#A]:module:source_files:
-** _module template_
-
-#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d"
-#+HEADER: :noweb yes
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-/++
- module source_read_source_files;<BR>
- - open markup files<BR>
- - if master file scan for addional files to import/insert
-+/
-module sisudoc.ocda.io_in.read_source_files;
-@safe:
-template spineRawMarkupContent() {
- import std.digest.sha;
- import std.file;
- import std.path;
- import sisudoc.ocda.meta;
- import sisudoc.ocda.io_in.paths_source;
- import sisudoc.ocda.meta.rgx_files;
- import sisudoc.ocda.meta.rgx;
- mixin spineRgxIn;
- static auto rgx = RgxI();
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- string[] _images=[];
- /+ ↓ the images a markup source shows.
- matched on the image markup itself ({ name.png }image, and the url and
- ~^ forms), ensure a filename-shaped word in prose is not mistaken for
- one, and over every match (so a line showing two images reports two).
- +/
- string[] _extract_images(S)(S content_block) {
- string[] images_;
- string _content_block = content_block.to!string;
- foreach (m; _content_block.matchAll(rgx.smid_image)) {
- images_ ~= m["image"].to!string;
- }
- return images_;
- }
- auto rawsrc = RawMarkupContent();
- struct ST_contents_inserts_images {
- char[][] contents;
- string[] insert_files;
- string[] images;
- }
- struct ST_header_content_inserts_images {
- char[] header;
- char[][] src_txt;
- string[] insert_files;
- string[] images;
- }
- struct ST_doc_digest {
- ubyte[32] markup_doc;
- ubyte[32] header;
- ubyte[32] text;
- }
- struct ST_doc_parts {
- char[] header_raw;
- char[][] sourcefile_body_content;
- string[] insert_file_list;
- string[] images_list;
- ST_doc_digest doc_digest;
- }
- ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) {
- ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct
- = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src);
- return _0_header_1_body_content_2_insert_filelist_struct;
- }
- struct RawMarkupContent {
- final sourceContent(in string fn_src) {
- auto raw = MarkupRawUnit();
- string source_txt_str
- = raw.markupSourceReadIn(fn_src);
- return source_txt_str;
- }
- final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)(
- O _opt_action,
- in string source_txt_str,
- in string fn_src=""
- ) {
- auto raw = MarkupRawUnit();
- string[] insert_file_list_get;
- string[] images_list_get;
- ST_header_content_inserts_images st
- = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str);
- char[] header_raw = st.header;
- char[][] sourcefile_body_content = st.src_txt;
- if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise
- auto ins = Inserts();
- ST_contents_inserts_images _cii
- = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
- sourcefile_body_content = _cii.contents;
- insert_file_list_get = _cii.insert_files.dup;
- images_list_get = _cii.images.dup;
- } else if (_opt_action.source_or_pod) {
- auto ins = Inserts();
- ST_contents_inserts_images _cii
- = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
- images_list_get = _cii.images.dup;
- } // image_list, if path could take sha256 digests already here?
- string header_type = "";
- ST_doc_digest dig;
- {
- dig.markup_doc = source_txt_str.sha256Of;
- dig.header = st.header.sha256Of;
- dig.text = sourcefile_body_content.sha256Of;
- }
- ST_doc_parts ret;
- {
- ret.header_raw = st.header;
- ret.sourcefile_body_content = sourcefile_body_content;
- ret.insert_file_list = insert_file_list_get;
- ret.images_list = images_list_get;
- ret.doc_digest = dig;
- }
- return ret;
- }
- }
- struct MarkupRawUnit {
- import std.digest.sha;
- import std.file;
- <<meta_markup_source_raw_read_file_source_string>>
- <<meta_markup_source_raw_doc_header_and_content_split>>
- <<meta_markup_source_raw_source_line_array>>
- <<meta_markup_source_raw_read_in_file>>
- <<meta_markup_source_raw_struct_of_header_and_body>>
- <<meta_markup_source_raw_get_insert_source_line_array>>
- }
- struct Inserts {
- struct ST_contents_and_images {
- char[][] insert_contents;
- string[] images;
- }
- ST_contents_and_images scan_subdoc_source(O)(
- O _opt_action,
- char[][] markup_sourcefile_insert_content,
- string fn_src
- ) {
- <<meta_inserts_scan>>
- foreach (line; markup_sourcefile_insert_content) {
- <<meta_inserts_scan_loop>>
- } // end src subdoc (inserts) loop
- <<meta_inserts_scan_post>>
- }
- ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)(
- O _opt_action,
- char[][] sourcefile_body_content,
- string fn_src
- ) {
- import std.algorithm;
- <<meta_master_doc_scan_for_insert_filenames>>
- foreach (line; sourcefile_body_content) {
- <<meta_master_doc_scan_for_insert_filenames_loop>>
- } // end src doc loop
- <<meta_master_doc_scan_for_insert_filenames_post>>
+ string abstractionSourceName(AbstractionSource _s) {
+ final switch (_s) {
+ case AbstractionSource.unknown: return "unknown";
+ case AbstractionSource.markup: return "markup source (.sst/.ssm)";
+ case AbstractionSource.pod: return "pod directory";
+ case AbstractionSource.pod_zip: return "pod zip";
+ case AbstractionSource.ssp: return ".ssp (abstraction as text)";
+ case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)";
}
}
-}
-#+END_SRC
-** get markup source, read file :source:markup:
-*** read file, source string [#A] :string:
-
-#+NAME: meta_markup_source_raw_read_file_source_string
-#+BEGIN_SRC d
-final private string readInMarkupSource(in char[] fn_src) {
- enforce(
- exists(fn_src) != 0,
- "file not found: «" ~
- fn_src ~ "»"
- );
- string source_txt_str;
- try {
- if (exists(fn_src)) {
- if (fn_src.getLinkAttributes.attrIsFile) {
- source_txt_str = fn_src.readText;
- } else {
- }
+ /+ ↓ what is this? by name, and for a directory by what it holds +/
+ AbstractionSource abstractionSourceOf(string _path) {
+ if (_path.length == 0) { return AbstractionSource.unknown; }
+ if (_path.isValidPath && _path.exists && _path.isDir) {
+ return (_path.chainPath("pod.manifest").array.exists)
+ ? AbstractionSource.pod : AbstractionSource.unknown;
}
- } catch (ErrnoException ex) {
- } catch (UTFException ex) {
- // Handle validation errors
- } catch (FileException ex) {
- // Handle errors
+ if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; }
+ if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; }
+ if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; }
+ if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; }
+ if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; }
+ return AbstractionSource.unknown;
}
- std.utf.validate(source_txt_str);
- return source_txt_str;
-}
-#+END_SRC
-
-*** document header & content, array.length == 2 [#A] :array:
-
-here you split document header and body, an array.length == 2
-split is on first match of level A~ (which is required)
-
-#+NAME: meta_markup_source_raw_doc_header_and_content_split
-#+BEGIN_SRC d
-@trusted final private char[][] header0Content1(in string src_text) { // cast(char[])
- /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/
- char[][] header_and_content;
- auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a);
- header_and_content ~= m.pre;
- header_and_content ~= m.hit ~ m.post;
- assert(header_and_content.length == 2,
- "document markup is broken, header body split == "
- ~ header_and_content.length.to!string
- ~ "; (header / body array split should == 2 (split is on level A~))"
- );
- return header_and_content;
-}
-#+END_SRC
-
-*** source line array :array:
-#+NAME: meta_markup_source_raw_source_line_array
-#+BEGIN_SRC d
-@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[])
- char[][] source_line_arr
- = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding);
- return source_line_arr;
-}
-#+END_SRC
-
-*** source content raw line array :array:
-- used for regular .sst files; master .ssm files and; .ssi inserts
-- regex is passed for relevant enforce match
-
-**** read in file
-
-#+NAME: meta_markup_source_raw_read_in_file
-#+BEGIN_SRC d
-string markupSourceReadIn(in string fn_src) {
- static auto rgx_files = RgxFiles();
- enforce(
- fn_src.match(rgx_files.src_pth_sst_or_ssm),
- "not a dr markup filename: «" ~
- fn_src ~ "»"
- );
- string source_txt_str = readInMarkupSource(fn_src);
- return source_txt_str;
-}
-#+END_SRC
-
-**** tuple (a) header, (b) body content, (c) file insert list & (d) image list?
-
-- header
-- body content
-- file insert list
-- [image list?]
-
-#+NAME: meta_markup_source_raw_struct_of_header_and_body
-#+BEGIN_SRC d
-ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) {
- string[] file_insert_list = [];
- string[] images_list = [];
- char[][] hc = header0Content1(source_txt_str);
- char[] header = hc[0];
- char[] source_txt = hc[1];
- char[][] source_line_arr = markupSourceLineArray(source_txt);
- ST_header_content_inserts_images ret;
- {
- ret.header = header;
- ret.src_txt = source_line_arr;
- ret.insert_files = file_insert_list;
- ret.images = images_list;
+ struct LoadedAbstraction {
+ AbstractionSource source;
+ string path;
+ bool loaded; // is .doc filled
+ string note; // why not, when it is not
+ SSPdocument doc;
}
- return ret;
-}
-#+END_SRC
-
-**** get insert source line array
-
-#+NAME: meta_markup_source_raw_get_insert_source_line_array
-#+BEGIN_SRC d
-final char[][] getInsertMarkupSourceContentRawLineArray(
- in char[] fn_src_insert,
- Regex!(char) rgx_file
-) {
- enforce(
- fn_src_insert.match(rgx_file),
- "not a dr markup filename: «" ~
- fn_src_insert ~ "»"
- );
- string source_txt_str = readInMarkupSource(fn_src_insert);
- char[][] source_line_arr = markupSourceLineArray(source_txt_str);
- return source_line_arr;
-}
-#+END_SRC
-
-** get markup source, master file & inserts :masterfile:inserts:
-
-- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
-
-*** scan inserts (sub-document) source :scan_insert_src:
-**** scan subdoc source
-
-#+NAME: meta_inserts_scan
-#+BEGIN_SRC d
-char[][] contents_insert;
-int code_block_status = 0;
-enum codeBlock { off, curly, tic, quotemarks }
-auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
-auto markup_src_file_path = fn_pth_full.captures[1];
-#+END_SRC
-**** loop insert (sub-document)
+ /+ ↓ load what can be loaded from the path alone.
-#+NAME: meta_inserts_scan_loop
-#+BEGIN_SRC d
-if (code_block_status == codeBlock.curly) {
- if (line.matchFirst(rgx.block_curly_code_close)) {
- code_block_status = codeBlock.off;
- }
- contents_insert ~= line;
-} else if (line.matchFirst(rgx.block_curly_code_open)) {
- code_block_status = codeBlock.curly;
- contents_insert ~= line;
-} else if (code_block_status == codeBlock.quotemarks) {
- if (line.matchFirst(rgx.block_quotemarks_close)) {
- code_block_status = codeBlock.off;
- }
- contents_insert ~= line;
-} else if (code_block_status == codeBlock.tic) {
- if (line.matchFirst(rgx.block_tic_close)) {
- code_block_status = codeBlock.off;
- }
- contents_insert ~= line;
-} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
- code_block_status = codeBlock.quotemarks;
- contents_insert ~= line;
-} else if (line.matchFirst(rgx.block_tic_code_open)) {
- code_block_status = codeBlock.tic;
- contents_insert ~= line;
-} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
- auto insert_fn = m.captures[2];
- auto insert_sub_pth = m.captures[1];
- auto fn_src_insert
- = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
- auto raw = MarkupRawUnit();
- auto markup_sourcesubfile_insert_content
- = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
- debug(insert_file) {
- writeln(line);
- writeln(fn_src_insert);
- writeln(
- " length contents insert array: ",
- markup_sourcesubfile_insert_content.length
- );
- }
- if (_opt_action.source_or_pod) {
- _images ~= _extract_images(markup_sourcesubfile_insert_content);
- }
- auto ins = Inserts();
- /+
- - 1. load file
- - 2. read lines
- - 3. scan lines
- - a. if filename insert, and insert filename
- - repeat 1
- - b. else
- - add line to new array;
- - build image list, search for any image files to add to image list
+ .ssp and .ocda.db come back filled. the three source forms come back
+ with loaded = false and a note saying where they are handled, because
+ reading them needs the manifest, environment and configuration that
+ spine.d builds, not a file path.
+/
-} else {
- contents_insert ~= line; // images to extract for image list?
- if (_opt_action.source_or_pod) {
- string[] _image_linelist = _extract_images(line);
- if (_image_linelist.length > 0) {
- _images ~= _image_linelist;
+ LoadedAbstraction abstractionLoad(string _path) {
+ LoadedAbstraction _out;
+ _out.path = _path;
+ _out.source = abstractionSourceOf(_path);
+ if (!_path.exists) {
+ _out.note = "no such file or directory";
+ return _out;
+ }
+ final switch (_out.source) {
+ case AbstractionSource.ssp:
+ _out.doc = sspReadFile(_path);
+ _out.loaded = (_out.doc.section_order.length > 0);
+ if (!_out.loaded) { _out.note = "no object sections found"; }
+ break;
+ case AbstractionSource.ocda_db:
+ _out.doc = dbReadFile(_path);
+ _out.loaded = (_out.doc.section_order.length > 0);
+ if (!_out.loaded) { _out.note = "no object sections found"; }
+ break;
+ case AbstractionSource.markup:
+ case AbstractionSource.pod:
+ case AbstractionSource.pod_zip:
+ _out.note = "a source form: read by the parser,"
+ ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the"
+ ~ " manifest, environment and configuration spine assembles";
+ break;
+ case AbstractionSource.unknown:
+ _out.note = "not a document source spine knows";
+ break;
}
+ return _out;
}
-}
-#+END_SRC
-
-**** post loop
-
-#+NAME: meta_inserts_scan_post
-#+BEGIN_SRC d
-ST_contents_and_images ret;
-{
- ret.insert_contents = contents_insert;
- ret.images = _images;
-}
-return ret;
-#+END_SRC
-
-*** scan document source :scan_src:
-**** scan doc source
-#+NAME: meta_master_doc_scan_for_insert_filenames
-#+BEGIN_SRC d
-char[][] contents;
-int code_block_status = 0;
-enum codeBlock { off, curly, tic, quotemarks }
-auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
-auto markup_src_file_path = fn_pth_full.captures[1];
-char[][] contents_insert;
-string[] _images =[];
-string[] insert_file_list =[];
-#+END_SRC
-
-**** include inserts: _loop master_ scan for inserts (insert documents)
-
-#+NAME: meta_master_doc_scan_for_insert_filenames_loop
-#+BEGIN_SRC d
-if (code_block_status == codeBlock.curly) {
- if (line.matchFirst(rgx.block_curly_code_close)) {
- code_block_status = codeBlock.off;
- }
- contents ~= line;
-} else if (line.matchFirst(rgx.block_curly_code_open)) {
- code_block_status = codeBlock.curly;
- contents ~= line;
-} else if (code_block_status == codeBlock.quotemarks) {
- if (line.matchFirst(rgx.block_quotemarks_close)) {
- code_block_status = codeBlock.off;
- }
- contents ~= line;
-} else if (code_block_status == codeBlock.tic) {
- if (line.matchFirst(rgx.block_tic_close)) {
- code_block_status = codeBlock.off;
- }
- contents ~= line;
-} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
- code_block_status = codeBlock.quotemarks;
- contents ~= line;
-} else if (line.matchFirst(rgx.block_tic_code_open)) {
- code_block_status = codeBlock.tic;
- contents ~= line;
-} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
- auto insert_fn = m.captures[2];
- auto insert_sub_pth = m.captures[1];
- auto fn_src_insert
- = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
- insert_file_list ~= fn_src_insert.to!string;
- auto raw = MarkupRawUnit();
- /+ TODO +/
- auto markup_sourcefile_insert_content
- = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
- debug(insert_file) {
- writeln(line);
- writeln(fn_src_insert);
- writeln(
- " length contents insert array: ",
- markup_sourcefile_insert_content.length
- );
- }
- auto ins = Inserts();
- ST_contents_and_images contents_insert_st = ins.scan_subdoc_source(
- _opt_action,
- markup_sourcefile_insert_content,
- fn_src_insert.to!string
- );
- contents ~= contents_insert_st.insert_contents;
- if (_opt_action.source_or_pod) {
- string[] _image_linelist = _extract_images(contents_insert_st.images);
- if (_image_linelist.length > 0) {
- _images ~= _image_linelist;
+ /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a
+ given artefact really does hold what it should +/
+ string[] abstractionLoadSummary(LoadedAbstraction _l) {
+ string[] _out;
+ _out ~= "source: " ~ abstractionSourceName(_l.source);
+ _out ~= "path: " ~ _l.path;
+ if (!_l.loaded) {
+ _out ~= "loaded: no";
+ if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; }
+ return _out;
}
- }
- /+
- - 1. load file
- - 2. read lines
- - 3. scan lines
- - a. if filename insert, and insert filename
- - repeat 1
- - b. else
- - add line to new array;
- - build image list, search for any image files to add to image list
- +/
-} else {
- contents ~= line;
- if (_opt_action.source_or_pod) {
- string[] _image_linelist = _extract_images(line);
- if (_image_linelist.length > 0) {
- _images ~= _image_linelist;
+ _out ~= "loaded: yes";
+ if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; }
+ if ("title.main" in _l.doc.meta) {
+ _out ~= "title: " ~ _l.doc.meta["title.main"];
+ }
+ if ("creator.author" in _l.doc.meta) {
+ _out ~= "author: " ~ _l.doc.meta["creator.author"];
+ }
+ _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, "
+ ~ _l.doc.make.length.to!string ~ " make, "
+ ~ _l.doc.doc_has.length.to!string ~ " doc_has";
+ int _objs, _citable, _headings;
+ string[] _sections;
+ foreach (section; _l.doc.section_order) {
+ int _n;
+ foreach (obj; _l.doc.abstraction[section]) {
+ ++_objs; ++_n;
+ if (obj.metainfo.ocn > 0) { ++_citable; }
+ if (obj.metainfo.is_a == "heading") { ++_headings; }
+ }
+ _sections ~= section ~ " " ~ _n.to!string;
}
+ _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string
+ ~ " citable, " ~ _headings.to!string ~ " headings)";
+ _out ~= "sections: " ~ _sections.join(", ");
+ return _out;
}
}
#+END_SRC
-**** post loop
-
-#+NAME: meta_master_doc_scan_for_insert_filenames_post
-#+BEGIN_SRC d
-string[] images = [];
-foreach(i; uniq(_images.sort())) {
- images ~= i;
-}
-debug(insert_file) {
- writeln(__LINE__);
- writeln(contents.length);
-}
-ST_contents_inserts_images ret;
-{
- ret.contents = contents;
- ret.insert_files = insert_file_list;
- ret.images = images;
-}
-return ret;
-#+END_SRC
-
-* C. read a .ssp file back into the document abstraction
-** _module template_ :module:ssp_files:
+* B. read a .ssp file back into the document abstraction - sisudoc.ocda.abstraction.ssp_in
#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp_in.d"
#+HEADER: :noweb yes
@@ -1154,8 +566,7 @@ template spineAbstractionRead() {
}
#+END_SRC
-* D. read a .db file into the document abstraction
-** _module template_ :module:db_files:
+* C. read a .db file into the document abstraction - sisudoc.ocda.abstraction.db_in
#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/db_in.d"
#+HEADER: :noweb yes
@@ -1454,175 +865,584 @@ template spineAbstractionDbRead() {
}
#+END_SRC
-* E. read a .db file into the document abstraction
-** _module template_ :module:db_files:
+* D. get doc from artefact - sisudoc.ocda.abstraction.doc_from_artefact
-#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d"
+#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
-module sisudoc.ocda.abstraction.load;
+module sisudoc.ocda.abstraction.doc_from_artefact;
@safe:
-/+ ↓ one way in, whatever the document is being read from
- spine's pipeline is markup -> abstraction -> output. once the abstraction
- is serialised, there is more than one thing an abstraction can be read
- from, and a consumer should not have to know which it was handed:
+/+ ↓ a document, from an artefact rather than from markup
+ spineAbstraction() parses markup and hands the output writers a doc:
+ an abstraction and a doc_matters. This does the same from a .ssp or a
+ .ocda.db, filling the same docMattersMake() arguments, so that what
+ comes out is the value the writers already take and they never learn
+ which of the two made it.
.
- .sst / .ssm + images the markup source
- pod (dir) + images the same, bundled
- pod .zip the same, zipped
- .ssp + images the abstraction, as text
- .ocda.db the abstraction, sqlite, images inside
- .
- A source may also be remote. A URL ending in .zip or in .ocda.db is
- downloaded to a temp file (downloadSourceUrl in
- ocda/io_in/read_zip_pod.d, guarded by rgx_url_source and by
- --allow-downloads) and the local path is processed in its place.
-
- Those two travel because each is self-sufficient: a pod holds its
- source and images, a .ocda.db holds its abstraction and images. A .ssp
- does not, describing its images by name and digest without carrying
- them, so it would arrive without them and is not fetched.
+ Three of the seven arguments come from the run and are the same either
+ way: program_info, opt_action, and the conf half of conf_make_meta,
+ which is site and run scoped (urls, papersize, the search database) and
+ has no business in a document's own artefact. The other four come from
+ the artefact:
.
- This module names those sources, tells them apart, and loads the two
- that are self-describing artefacts, returning the value the parser
- produces.
+ conf_make_meta meta and make from the @meta and @make blocks, over
+ the site config the run assembled
+ doc_has docHasFromAbstraction, over the objects
+ doc_digest the markup digest from @source
+ manifest synthesised from the artefact's path and the
+ document's own name and language
.
- The three source forms are deliberately *not* loaded here. They are the
- parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs
- the environment, the options and the configuration that spine.d
- assembles, none of which belongs in a loader. What this module gives
- that case is the dispatch and a plain statement of where to go.
+ What is not here, deliberately: insert_file_list, which is read only by
+ source_pod.d for --source and --pod2. No artefact carries the markup,
+ so a pod cannot be rebuilt from one, and the empty list is the truthful
+ answer rather than a gap.
+/
-template spineAbstractionLoad() {
+template spineDocFromArtefact() {
import std.algorithm : endsWith;
+ import std.array;
import std.conv : to;
- import std.file;
import std.path;
+ import std.process : thisProcessID;
+ import std.regex;
import std.stdio;
import std.string;
- import sisudoc.ocda.abstraction.db_in;
- mixin spineAbstractionDbRead; // brings the .ssp reader with it
-
- enum AbstractionSource {
- unknown,
- markup, // .sst or .ssm, with its images beside it
- pod, // a directory holding pod.manifest
- pod_zip, // that directory, zipped
- ssp, // .ssp, the abstraction as text, images beside it
- ocda_db, // .ocda.db, the abstraction as sqlite, images inside it
+ import sisudoc.ocda.meta.conf_make_meta_structs;
+ import sisudoc.ocda.meta.doc_matters;
+ import sisudoc.ocda.io_in.paths_source;
+ import sisudoc.ocda.abstraction.doc_has;
+ import sisudoc.ocda.abstraction.load;
+ import sisudoc.ocda.meta.topic_register;
+ mixin spineDocMatters;
+ mixin spineDocHasFromAbstraction;
+ mixin spineAbstractionLoad;
+ mixin spineTopicRegister;
+ /+ ↓ a .ssp header key from a field name: the first underscore becomes the
+ dot that separates the group, so title_main is title.main and
+ rights_copyright_text is rights.copyright_text. The writer's keys are
+ formed the same way, which is what lets this be read off the struct
+ rather than kept as a second list to fall out of step with it.
+ +/
+ private string _sspKeyOf(string _member) {
+ foreach (i, c; _member) {
+ if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; }
+ }
+ return _member;
}
-
- string abstractionSourceName(AbstractionSource _s) {
- final switch (_s) {
- case AbstractionSource.unknown: return "unknown";
- case AbstractionSource.markup: return "markup source (.sst/.ssm)";
- case AbstractionSource.pod: return "pod directory";
- case AbstractionSource.pod_zip: return "pod zip";
- case AbstractionSource.ssp: return ".ssp (abstraction as text)";
- case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)";
+ /+ ↓ the document's own header, over the site config the run assembled.
+ every string field of MetaComposite is looked for under its own key,
+ so a field added there and emitted by the writer arrives here without
+ anything being added. The arr fields are split from the string they
+ come from, as agreed: that is parsing a value, not deriving a
+ structure.
+ +/
+ ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) {
+ ConfComposite _out = _site_conf; // conf: site and run scoped, kept
+ static foreach (_m; __traits(allMembers, MetaComposite)) {
+ static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{
+ enum string _key = _sspKeyOf(_m);
+ if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; }
+ }}
+ }
+ /+ ↓ title_sub is a copy of title_subtitle, made where the header is
+ read, and is what epub3 puts in dc:title id="subtitle". It is not
+ emitted, being the same string twice; it is remade here the way the
+ yaml reader makes it +/
+ _out.meta.title_sub = _out.meta.title_subtitle;
+ /+ ↓ creator.author is the author list joined with ", ", so splitting on
+ it gives the list back +/
+ if (_out.meta.creator_author.length > 0) {
+ foreach (_a; _out.meta.creator_author.split(", ")) {
+ if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; }
+ }
+ }
+ /+ ↓ the topic register arrays, by the one rule the yaml reader uses:
+ the split is not a plain one and deriving it twice would be two
+ answers to the same question +/
+ {
+ auto _tr = topicRegisterArrays(_out.meta.classify_topic_register);
+ _out.meta.classify_topic_register_arr = _tr.arr;
+ _out.meta.classify_topic_register_expanded_arr = _tr.expanded;
+ }
+ /+ ↓ make: the keys are the field names, with no group to separate +/
+ if (auto _v = "doc_type" in _ssp.make) {
+ _out.make.doc_type = *_v; _out.make_str.doc_type = *_v;
+ }
+ if (auto _v = "auto_num_top_at_level" in _ssp.make) {
+ _out.make.auto_num_top_at_level = *_v;
+ }
+ if (auto _v = "auto_num_top_lv" in _ssp.make) {
+ try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {}
+ }
+ if (auto _v = "auto_num_depth" in _ssp.make) {
+ try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {}
+ }
+ if (auto _v = "breaks" in _ssp.make) {
+ _out.make.breaks = *_v; _out.make_str.breaks = *_v;
+ }
+ if (auto _v = "home_button_text" in _ssp.make) {
+ _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v;
+ }
+ /+ ↓ footer is a list, joined with the separator the format escapes +/
+ if (auto _v = "footer" in _ssp.make) {
+ _out.make.footer = sspUnescapeSplit(*_v);
+ _out.make_str.footer = _out.make.footer;
}
+ return _out;
}
-
- /+ ↓ what is this? by name, and for a directory by what it holds +/
- AbstractionSource abstractionSourceOf(string _path) {
- if (_path.length == 0) { return AbstractionSource.unknown; }
- if (_path.isValidPath && _path.exists && _path.isDir) {
- return (_path.chainPath("pod.manifest").array.exists)
- ? AbstractionSource.pod : AbstractionSource.unknown;
+ /+ ↓ where a document loaded from an artefact says it is.
+ an artefact names its document (% Source:) and its language (@source
+ language) but not where the pod holding its images sits, so that is
+ taken from where the artefact itself was found:
+ .
+ pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/
+ pod/<uid>.ocda.db -> pod/<doc>/
+ .
+ which is where --pod2 puts them both. The images beside a .ssp are
+ then found where html.d already looks; a .ocda.db carries its own and
+ does not need the directory to exist.
+ +/
+ struct ST_ArtefactPaths {
+ string pod_dir;
+ string src_file_with_path;
+ }
+ ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) {
+ ST_ArtefactPaths _out;
+ string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en";
+ string _dir = _artefact.dirName;
+ if (_artefact.endsWith(".ssp")) {
+ /+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/
+ _out.pod_dir = (_dir.baseName == "abstraction")
+ ? _dir.dirName.dirName : _dir;
+ } else {
+ /+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/
+ string _uid = _artefact.baseName;
+ foreach (_sfx; [".ocda.db", ".db"]) {
+ if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; }
+ }
+ if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; }
+ _out.pod_dir = (_dir.chainPath(_uid).array).to!string;
}
- if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; }
- if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; }
- if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; }
- if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; }
- if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; }
- return AbstractionSource.unknown;
+ _out.src_file_with_path = (_out.pod_dir
+ .chainPath("media").chainPath("text").chainPath(_lang)
+ .chainPath(_ssp.source).array).to!string;
+ return _out;
}
-
- struct LoadedAbstraction {
- AbstractionSource source;
- string path;
- bool loaded; // is .doc filled
- string note; // why not, when it is not
- SSPdocument doc;
+ /+ ↓ a database's images, written where the writers look for them.
+ returns the pod directory to use in place of the one beside the
+ artefact, or "" when the database carries no images and the pod
+ beside it should stand. Each blob is checked against the sha256 the
+ database recorded with it: the two were written together, so a
+ mismatch means the file is damaged, and saying so is the point of
+ having recorded the digest at all.
+ +/
+ @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) {
+ import std.digest : toHexString;
+ import std.digest.sha : sha256Of;
+ import std.file : mkdirRecurse, write, tempDir;
+ import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead;
+ mixin spineAbstractionDbRead _dbr;
+ auto _files = _dbr.dbReadFiles(_artefact, "image");
+ if (_files.length == 0) { return ""; }
+ string _root = (tempDir.chainPath("spine-ocda-"
+ ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string;
+ string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string;
+ try {
+ _img_dir.mkdirRecurse;
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not make an image directory for ", _artefact,
+ ": ", ex.msg);
+ return "";
+ }
+ foreach (_f; _files) {
+ string _got = _f.data.sha256Of.toHexString.to!string;
+ if (_f.sha256.length > 0 && _got != _f.sha256) {
+ stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName,
+ " does not match the digest recorded with it (", _f.sha256, " expected, ",
+ _got, " found); it is written out as it stands");
+ }
+ try {
+ (_img_dir.chainPath(_f.name).array).to!string.write(_f.data);
+ } catch (Exception ex) {
+ stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg);
+ }
+ }
+ if (_opt_action.vox_gt_1) {
+ writeln(" ", _files.length, " image(s) from the database, in ", _img_dir);
+ }
+ return _root;
}
-
- /+ ↓ load what can be loaded from the path alone.
-
- .ssp and .ocda.db come back filled. the three source forms come back
- with loaded = false and a note saying where they are handled, because
- reading them needs the manifest, environment and configuration that
- spine.d builds, not a file path.
+ /+ ↓ the images a .ssp describes, against the ones beside it.
+ a .ssp carries no image bytes, so what it can do is say whether the
+ files in the pod it sits in are the ones the abstraction was built
+ from. Nothing checked that before: a changed or wrong image in the
+ directory paired silently.
+/
- LoadedAbstraction abstractionLoad(string _path) {
- LoadedAbstraction _out;
- _out.path = _path;
- _out.source = abstractionSourceOf(_path);
- if (!_path.exists) {
- _out.note = "no such file or directory";
- return _out;
+ @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) {
+ import std.digest : toHexString;
+ import std.digest.sha : sha256Of;
+ import std.file : exists, read;
+ string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string;
+ int _checked, _missing, _wrong;
+ foreach (_section; _ssp.section_order) {
+ foreach (obj; _ssp.abstraction[_section]) {
+ foreach (_img; obj.metainfo.sha256.images) {
+ if (_img.fileName.length == 0) { continue; }
+ string _pth = (_img_dir.chainPath(_img.fileName).array).to!string;
+ if (!_pth.exists) { ++_missing; continue; }
+ ++_checked;
+ string _want = _img.fileHash_sha256.toHexString.to!string;
+ string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string;
+ if (_want != _got && _want != "00000000000000000000000000000000"
+ ~ "00000000000000000000000000000000") {
+ ++_wrong;
+ stderr.writeln("WARNING: image ", _img.fileName, " beside ",
+ _ssp.source, " is not the one the abstraction was built from");
+ }
+ }
+ }
}
- final switch (_out.source) {
- case AbstractionSource.ssp:
- _out.doc = sspReadFile(_path);
- _out.loaded = (_out.doc.section_order.length > 0);
- if (!_out.loaded) { _out.note = "no object sections found"; }
- break;
- case AbstractionSource.ocda_db:
- _out.doc = dbReadFile(_path);
- _out.loaded = (_out.doc.section_order.length > 0);
- if (!_out.loaded) { _out.note = "no object sections found"; }
- break;
- case AbstractionSource.markup:
- case AbstractionSource.pod:
- case AbstractionSource.pod_zip:
- _out.note = "a source form: read by the parser,"
- ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the"
- ~ " manifest, environment and configuration spine assembles";
- break;
- case AbstractionSource.unknown:
- _out.note = "not a document source spine knows";
- break;
+ if (_missing > 0) {
+ stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are",
+ " not in ", _img_dir);
+ }
+ if (_opt_action.vox_gt_1 && _checked > 0) {
+ writeln(" ", _checked, " image(s) checked against the abstraction's digests",
+ (_wrong == 0) ? ", all matching" : "");
+ }
+ }
+ /+ ↓ the doc the output writers take, from an artefact.
+ the reading is done here rather than by the caller, and that is not
+ only tidiness: main() is @system, so a template mixed in there infers
+ @system for everything it declares, and the output writers are @safe
+ and will not take objects of such a type. Doing it inside this module,
+ which is @safe:, is what makes the loaded objects the same kind of
+ thing the parser's are. Called as spineDocFromArtefact!()(...), the
+ way spineAbstraction!() is, rather than mixed in.
+ +/
+ auto spineDocFromArtefact(P,O,E)(
+ string _artefact,
+ P program_info,
+ O _opt_action,
+ E _env,
+ ConfComposite _site_conf,
+ ) {
+ auto _loaded = abstractionLoad(_artefact);
+ auto _ssp = _loaded.doc;
+ auto _pths = artefactPaths(_artefact, _ssp);
+ /+ ↓ the images.
+ a .ocda.db carries its own, which is what lets it travel on its
+ own, so they are taken from it and written where the output writers
+ look for images: a directory of this run's making, not the pod
+ beside the artefact, which may be somebody's published tree. A .ssp
+ only describes its images, so those are the ones in the pod it sits
+ in, and are checked against the digests it recorded.
+ +/
+ string _images_tmp;
+ if (_loaded.loaded) {
+ if (_artefact.endsWith(".ssp")) {
+ _imagesVerify(_pths.pod_dir, _ssp, _opt_action);
+ } else {
+ string _extracted = _imagesExtract(_artefact, _opt_action);
+ if (_extracted.length > 0) {
+ /+ ↓ the source path has to move with it: image_dir_path is
+ reached from the document's own file, not from the pod +/
+ _images_tmp = _extracted;
+ _pths.pod_dir = _extracted;
+ _pths.src_file_with_path = (_extracted
+ .chainPath("media").chainPath("text")
+ .chainPath(("language" in _ssp.source_info)
+ ? _ssp.source_info["language"] : "en")
+ .chainPath(_ssp.source).array).to!string;
+ }
+ }
+ }
+ auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path);
+ auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf);
+ auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action);
+ /+ ↓ the digests: the markup digest is what @source carries, and it is
+ the one that says which source this abstraction was made from. The
+ header and text digests are of parts an artefact does not keep
+ separately, and are left unset rather than guessed at
+ +/
+ ST_ArtefactDigest _dig;
+ if (auto _v = "digest" in _ssp.source_info) {
+ _dig.markup_doc = _hexToDigest(*_v);
+ }
+ auto _abstraction = _ssp.abstraction;
+ auto _matters = docMattersMake(
+ program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init,
+ );
+ /+ ↓ the same shape spineAbstraction() returns, built the same way: a
+ struct of accessors over the locals, with whether the artefact
+ could be read at all, which the caller has to know +/
+ auto theDOC() {
+ struct ST_DOC {
+ const auto abstraction() { return _abstraction; }
+ auto matters() { return _matters; }
+ bool loaded() { return _loaded.loaded; }
+ string images_tmp() { return _images_tmp; }
+ string note() { return _loaded.note; }
+ string source_name() { return abstractionSourceName(_loaded.source); }
+ }
+ return ST_DOC();
+ }
+ return theDOC();
+ }
+ /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from
+ an artefact, @source having recorded it; the header and text digests
+ are of parts no artefact keeps separately and are left unset rather
+ than guessed at. Shaped to match ST_doc_digest, which is declared
+ inside spineRawMarkupContent and reachable only by parsing a file.
+ +/
+ struct ST_ArtefactDigest {
+ ubyte[32] markup_doc;
+ ubyte[32] header;
+ ubyte[32] text;
+ }
+ private ubyte[32] _hexToDigest(string _hex) {
+ ubyte[32] _out;
+ if (_hex.length < 64) { return _out; }
+ foreach (i; 0..32) {
+ try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {}
}
return _out;
}
+}
+#+END_SRC
- /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a
- given artefact really does hold what it should +/
- string[] abstractionLoadSummary(LoadedAbstraction _l) {
- string[] _out;
- _out ~= "source: " ~ abstractionSourceName(_l.source);
- _out ~= "path: " ~ _l.path;
- if (!_l.loaded) {
- _out ~= "loaded: no";
- if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; }
- return _out;
+* E. Get OCD
+
+get abstracted objects for downstream processing
+
+#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_has.d"
+#+HEADER: :noweb yes
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module sisudoc.ocda.abstraction.doc_has;
+@safe:
+/+ ↓ ST_DocHas from a loaded abstraction
+
+ the parser builds ST_DocHas as it goes, with the markup in front of it.
+ A document loaded from a .ssp or a .ocda.db has to arrive at the same
+ value from what the artefact carries, and this is where that is done.
+
+ Every one of these is an *index over properties the artefact already
+ holds*, not a re-derivation of something it does not:
+
+ imagelist the .image records' filenames
+ segnames_lv4 .anchor of each body heading at level 4
+ segnames_lv_0_to_4 .segment_epub of each heading at level 4 or less
+ tag_associations .anchor, .anchor_tag and .segment_* per object
+ section_keys_sequenced which sections are non-empty, plus the run's
+ own output flags
+
+ That distinction is the line the .ssp skill draws: assembling an index
+ is fine and cannot drift, because a wrong index would not survive the
+ round trip that produced its inputs; recomputing dom_status or ancestors
+ would be re-derivation and is exactly what ocda exists to remove. If a
+ value is not in the file and cannot be indexed out of what is, it must
+ be added to the artefact instead of reconstructed here.
+
+ The counts come from the @doc_has block rather than being recounted, for
+ the same reason.
++/
+template spineDocHasFromAbstraction() {
+ import std.algorithm : sort, uniq;
+ import std.array;
+ import std.conv : to;
+ import std.json : JSONValue;
+ import std.regex;
+ import sisudoc.ocda.meta.metadoc_object_setter;
+ import sisudoc.ocda.meta.rgx;
+ mixin ObjectSetter;
+ mixin spineRgxIn;
+ /+ ↓ the sections, in the order a document holds them +/
+ enum string[] doc_sections = [
+ "head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb", "tail",
+ ];
+ /+ ↓ the tail sections that become segments of their own, in the order
+ after_doc_determine_segnames appends them +/
+ enum string[] tail_sections = [
+ "endnotes", "glossary", "bibliography", "bookindex", "blurb",
+ ];
+ private uint _count(string[string] _doc_has, string _key) {
+ if (auto _v = _key in _doc_has) {
+ try { return (*_v).to!uint; } catch (Exception ex) { return 0; }
}
- _out ~= "loaded: yes";
- if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; }
- if ("title.main" in _l.doc.meta) {
- _out ~= "title: " ~ _l.doc.meta["title.main"];
+ return 0;
+ }
+ private bool _has_section(A)(A _abst, string _section) {
+ if (auto _s = _section in _abst) { return (*_s).length > 1; }
+ return false;
+ }
+ /+ ↓ which document sections each output format walks, and in what order.
+
+ the same rule the parser applies: the three that are always there,
+ then each tail section that the document actually has, then "tail"
+ for the formats that close with one. It depends on the run as well as
+ on the document, since only html and epub take the tail.
+ +/
+ string[][string] sectionKeysSequenced(A,O)(A _abst, O _opt_action) {
+ string[][string] _keys = [
+ "scroll": ["head", "toc", "body",],
+ "seg": ["head", "toc", "body",],
+ "sql": ["head", "body",],
+ "latex": ["head", "toc", "body",],
+ "text": ["head", "toc", "body",],
+ ];
+ if (_has_section(_abst, "endnotes")) {
+ _keys["scroll"] ~= "endnotes";
+ _keys["seg"] ~= "endnotes";
+ _keys["latex"] ~= "endnotes";
+ _keys["text"] ~= "endnotes";
}
- if ("creator.author" in _l.doc.meta) {
- _out ~= "author: " ~ _l.doc.meta["creator.author"];
+ if (_has_section(_abst, "glossary")) {
+ foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "glossary"; }
}
- _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, "
- ~ _l.doc.make.length.to!string ~ " make, "
- ~ _l.doc.doc_has.length.to!string ~ " doc_has";
- int _objs, _citable, _headings;
- string[] _sections;
- foreach (section; _l.doc.section_order) {
- int _n;
- foreach (obj; _l.doc.abstraction[section]) {
- ++_objs; ++_n;
- if (obj.metainfo.ocn > 0) { ++_citable; }
- if (obj.metainfo.is_a == "heading") { ++_headings; }
+ if (_has_section(_abst, "bibliography")) {
+ foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bibliography"; }
+ }
+ if (_has_section(_abst, "bookindex")) {
+ foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bookindex"; }
+ }
+ if (_has_section(_abst, "blurb")) {
+ foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "blurb"; }
+ }
+ if (_opt_action.html_scroll || _opt_action.html_seg || _opt_action.epub) {
+ _keys["scroll"] ~= "tail";
+ _keys["seg"] ~= "tail";
+ }
+ return _keys;
+ }
+ /+ ↓ the whole of it +/
+ ST_DocHas docHasFromAbstraction(A,O)(
+ A _abst,
+ string[string] _doc_has_block,
+ O _opt_action,
+ ) {
+ static auto rgx = RgxI();
+ ST_DocHas _out;
+ _out.inline_links = _count(_doc_has_block, "inline_links");
+ _out.inline_notes_reg = _count(_doc_has_block, "inline_notes_reg");
+ _out.inline_notes_star = _count(_doc_has_block, "inline_notes_star");
+ _out.codeblocks = _count(_doc_has_block, "codeblocks");
+ _out.tables = _count(_doc_has_block, "tables");
+ _out.blocks = _count(_doc_has_block, "blocks");
+ _out.groups = _count(_doc_has_block, "groups");
+ _out.poems = _count(_doc_has_block, "poems");
+ _out.quotes = _count(_doc_has_block, "quotes");
+ string[] _images;
+ string[string][string] _tag_assoc;
+ /+ ↓ segnames["html"] is seeded with the toc, which is a segment of its
+ own before any level 4 heading opens one +/
+ _out.segnames_lv4 ~= "toc";
+ foreach (_section; doc_sections) {
+ if (_section !in _abst) { continue; }
+ /+ ↓ head is always walked; every other section only when it holds
+ more than the placeholder object, which is the guard the parser
+ puts on each of its section loops
+ +/
+ if (_section != "head" && !_has_section(_abst, _section)) { continue; }
+ foreach (obj; _abst[_section]) {
+ /+ ↓ the images this object names, as the abstraction recorded them +/
+ foreach (_img; obj.metainfo.sha256.images) { _images ~= _img.fileName; }
+ /+ ↓ every object: its html anchor is in a segment, and its epub
+ segment anchor stands for itself +/
+ if (obj.tags.anchor_tag_html.length > 0) {
+ _tag_assoc[obj.tags.anchor_tag_html]["seg_lv4"] = obj.tags.in_segment_html;
+ }
+ if (obj.tags.segment_anchor_tag_epub.length > 0) {
+ _tag_assoc[obj.tags.segment_anchor_tag_epub]["seg_lv1to4"]
+ = obj.tags.segment_anchor_tag_epub;
+ }
+ /+ ↓ a heading's own anchor tags, and the anchors declared inline in
+ an object's text, both point at the segment the object is in +/
+ /+ ↓ a heading above level 4 opens no html segment of its own, so a
+ cross reference to it has to land on the level 4 segment that
+ follows it. .segment is exactly that (the segment a heading
+ opens or falls into), where .segment_html_is is the segment an
+ object sits in; epub segments go down to level 1, so there
+ .segment_epub stands for the heading itself
+ +/
+ if (obj.tags.heading_lev_anchor_tag.length > 0
+ && obj.tags.in_segment_html.length > 0
+ && obj.metainfo.heading_lev_markup >= 4
+ ) {
+ _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv4"]
+ = obj.tags.in_segment_html;
+ if (obj.tags.segment_anchor_tag_epub.length > 0) {
+ _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv1to4"]
+ = obj.tags.segment_anchor_tag_epub;
+ }
+ }
+ foreach (m; obj.text.matchAll(rgx.inline_link_anchor)) {
+ string _a = m["anchor"].to!string;
+ if (_a.length == 0 || _a in _tag_assoc) { continue; }
+ _tag_assoc[_a]["seg_lv4"] = obj.tags.html_segment_anchor_tag_is;
+ _tag_assoc[_a]["seg_lv1to4"] = obj.tags.epub_segment_anchor_tag_is;
+ }
+ /+ ↓ the segment name lists, in document order +/
+ if (obj.metainfo.is_a == "heading") {
+ if (obj.metainfo.heading_lev_markup <= 4) {
+ _out.segnames_lv_0_to_4 ~= obj.tags.segment_anchor_tag_epub;
+ }
+ if (_section == "body" && obj.metainfo.heading_lev_markup == 4) {
+ _out.segnames_lv4 ~= obj.tags.anchor_tag_html;
+ }
+ /+ ↓ a heading above level 4 opens no html segment of its own, so
+ a cross reference to it lands on the level 4 segment that
+ follows. That is .segment_lv4_is, resolved in ocda where the
+ finished sections are to hand and carried by the artefact, so
+ there is nothing to work out here
+ +/
+ if (obj.tags.segment_lv4_is.length > 0) {
+ /+ ↓ seg_lv4 only, and not through heading_lev_anchor_tag: that
+ names the level 4 heading itself, whose own entry says it
+ opens that segment rather than merely sits in it. epub
+ segments go down to level 1, so seg_lv1to4 is untouched
+ +/
+ foreach (_k; [obj.metainfo.identifier, obj.tags.anchor_tag_html]) {
+ if (_k.length > 0) { _tag_assoc[_k]["seg_lv4"] = obj.tags.segment_lv4_is; }
+ }
+ }
+ }
}
- _sections ~= section ~ " " ~ _n.to!string;
}
- _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string
- ~ " citable, " ~ _headings.to!string ~ " headings)";
- _out ~= "sections: " ~ _sections.join(", ");
+ foreach (_section; tail_sections) {
+ if (_has_section(_abst, _section)) { _out.segnames_lv4 ~= _section; }
+ }
+ /+ ↓ the body's objects, in a pass of their own after every anchor above
+ has been recorded, because the parser does it in that order and the
+ order is what decides two of these entries.
+ .
+ each body object's identifier maps to the segment it is in. seg_lv4
+ is set only where it is not already, an anchor of the same name
+ having said it *names* a segment rather than sits in one; seg_lv1to4
+ is set always. A heading's own anchor can be a bare number taken
+ from its text ("2. Shorter Terms" gives the anchor "2"), which then
+ collides with the ocn of an unrelated object, and this is the order
+ that resolves the collision the way the parser resolves it: in
+ free_culture, ocn 2 keeps seg_lv4 "them" from the anchor and takes
+ seg_lv1to4 "_part_1" from itself.
+ +/
+ if (_has_section(_abst, "body")) {
+ foreach (obj; _abst["body"]) {
+ if (obj.metainfo.identifier.length == 0) { continue; }
+ if (!((obj.metainfo.identifier in _tag_assoc)
+ && ("seg_lv4" in _tag_assoc[obj.metainfo.identifier]))
+ ) {
+ _tag_assoc[obj.metainfo.identifier]["seg_lv4"]
+ = obj.tags.html_segment_anchor_tag_is;
+ }
+ _tag_assoc[obj.metainfo.identifier]["seg_lv1to4"]
+ = obj.tags.epub_segment_anchor_tag_is;
+ }
+ }
+ _out.imagelist = _images.sort.uniq.array;
+ _out.tag_associations = _tag_assoc;
+ _out.section_keys_sequenced = sectionKeysSequenced(_abst, _opt_action);
return _out;
}
}
diff --git a/org/in_markup_source_files.org b/org/in_markup_source_files.org
new file mode 100644
index 0000000..1b8cee0
--- /dev/null
+++ b/org/in_markup_source_files.org
@@ -0,0 +1,801 @@
+-*- mode: org -*-
+#+TITLE: sisudoc spine (doc_reform) markup source raw
+#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
+#+FILETAGS: :spine:sourcefile:read:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
+
+* A. get _config file_, read in (.readText) [#A]
+** _module template_ :module:config_files:
+
+#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d"
+#+HEADER: :noweb yes
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ read configuration files<BR>
+ - read config files<BR>
+ meta_config_files.d
++/
+module sisudoc.ocda.io_in.read_config_files;
+@safe:
+import std.file;
+import std.path;
+import sisudoc.ocda.meta;
+import sisudoc.ocda.io_in.paths_source;
+import sisudoc.ocda.meta.rgx_files;
+import sisudoc.ocda.meta.rgx;
+<<meta_config_file_hub_read_site_config>>
+<<meta_config_file_hub_read_document_config>>
+<<meta_config_file_hub_read_site_yaml>>
+#+END_SRC
+
+*** read config files (config local site & dr document make) (yaml)
+**** site configuration
+
+SEE NOTES on configuration hierarchy in spine.org
+
+#+NAME: meta_config_file_hub_read_site_config
+#+BEGIN_SRC d
+template readConfigSite() {
+ @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) {
+ mixin spineRgxIn;
+ static auto rgx = RgxI();
+ string conf_filename = "NONE";
+ string config_file_str;
+ string default_config_file_str = format(q"┃
+flag:
+ act0: "--html"
+ act1: "--html --epub"
+output:
+ path: "%s"
+default:
+ language: "en"
+ papersize: "a4"
+ text_wrap: "80"
+ digest: "sha256"
+webserv:
+ http: "%s"
+ host: "%s"
+ data_http: "%s"
+ data_host: "%s"
+ data_root_url: "%s"
+ data_root_path: "%s"
+ data_root_part: ""
+ images_root_part: "image"
+ cgi_search_form_title: "%s"
+ cgi_http: "%s"
+ cgi_host: "%s"
+ cgi_bin_url: "%s"
+ cgi_bin_subpath: "%s"
+ cgi_bin_path: "%s"
+ cgi_search_script: "%s"
+ cgi_port: ""
+ cgi_user: ""
+ cgi_action: "%s"
+ db_sqlite_path: "%s"
+ db_sqlite_filename: "%s"
+ db_pg_table: ""
+ db_pg_user: ""
+┃",
+ _cfg.processing_path_doc_root, // doc root
+ _cfg.http_request_type, // http
+ _cfg.http_host, // host / domain
+ _cfg.http_request_type, // data "http" or "https"
+ _cfg.http_host, // data domain "localhost"
+ _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org"
+ _cfg.processing_path_doc_root, // data root path
+ _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search"
+ _cfg.http_request_type, // cgi http
+ _cfg.http_host, // cgi host
+ _cfg.cgi_url_root, // cgi bin url
+ _cfg.cgi_bin_subpath, // cgi bin path
+ _cfg.cgi_bin_root, // cgi bin path
+ _cfg.cgi_filename, // cgi filename
+ _cfg.cgi_url_action, // cgi action
+ _cfg.db_sqlite_path, // sqlite db path
+ _cfg.db_sqlite_filename, // sqlite db filename
+);
+ foreach(conf_fn; [_conf_file_details.config_filename_site]) {
+ foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) {
+ char[] conf_file;
+ conf_filename = conf_fn;
+ if (exists(pth)) {
+ auto f_attrib = pth.getLinkAttributes;
+ if (
+ _conf_file_details.possible_config_path_locations.config_local_site.length == 1
+ && f_attrib.attrIsFile
+ ) {
+ conf_file = pth.to!(char[]);
+ conf_filename = pth.baseName;
+ } else if (f_attrib.attrIsDir) {
+ conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
+ conf_filename = conf_fn;
+ }
+ try {
+ if (exists(conf_file)) {
+ if (conf_file.getLinkAttributes.attrIsFile) {
+ if (_opt_action.vox_gt_2 || _opt_action.debug_do) {
+ writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)");
+ }
+ config_file_str = conf_file.readText;
+ break;
+ }
+ }
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
+ }
+ }
+ }
+ if (config_file_str.length > 0) { break; }
+ }
+ if (config_file_str.length > 0) {
+ import dyaml;
+ Node yaml_root;
+ try {
+ yaml_root = Loader.fromString(config_file_str).load();
+ } catch (Throwable) {
+ import std.stdio;
+ writeln("ERROR failed to read config file content, not parsed as yaml, program default used");
+ conf_filename = "VIRTUAL";
+ config_file_str = default_config_file_str;
+ }
+ }
+ if (config_file_str.length == 0) { /+ use dummy default config file +/
+ // writeln("WARNING config file NOT found, default provided");
+ conf_filename = "VIRTUAL";
+ config_file_str = default_config_file_str;
+ }
+ struct _ConfContent {
+ string filename() {
+ return conf_filename;
+ }
+ string filetype() {
+ string _ft = "";
+ if (content.match(rgx.yaml_config)) {
+ _ft = "yaml";
+ }
+ return _ft;
+ }
+ string content() {
+ return config_file_str;
+ }
+ }
+ return _ConfContent();
+ }
+}
+#+END_SRC
+
+**** document make/config
+
+#+NAME: meta_config_file_hub_read_document_config
+#+HEADER: :noweb yes
+#+BEGIN_SRC d
+static template readConfigDoc() {
+ import std.file;
+ import std.path;
+ import sisudoc.ocda.meta;
+ import sisudoc.ocda.io_in.paths_source;
+ import sisudoc.ocda.meta.rgx_files;
+ import sisudoc.ocda.meta.rgx;
+ @system final auto readConfigDoc(M,E)(M _manifested, E _env) {
+ mixin spineRgxIn;
+ static auto rgx = RgxI();
+ mixin spineRgxFiles;
+ static auto rgx_files = RgxFiles();
+ string config_file_str;
+ string conf_filename = "NONE";
+ auto _conf_file_details = configFilePaths!()(_manifested, _env);
+ string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make;
+ foreach(conf_fn; [_conf_file_details.config_filename_document]) {
+ foreach(pth; possible_config_path_locations) {
+ char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
+ conf_filename = conf_fn;
+ if (config_file_str.length > 0) {
+ break;
+ }
+ try {
+ if (exists(conf_file)) {
+ if (conf_file.getLinkAttributes.attrIsFile) {
+ config_file_str = conf_file.readText;
+ break;
+ }
+ }
+ } catch (ErrnoException ex) {
+ } catch (FileException ex) {
+ }
+ }
+ if (config_file_str.length > 0) { break; }
+ }
+ struct _ConfContent {
+ string filename() {
+ return conf_filename;
+ }
+ string content() {
+ return config_file_str;
+ }
+ string filetype() {
+ string _ft = "";
+ if (content.match(rgx.yaml_config)) {
+ _ft = "yaml";
+ }
+ return _ft;
+ }
+ }
+ return _ConfContent();
+ }
+}
+#+END_SRC
+
+*** YAML config (config local site & dr document make) :file:config:hub:
+
+#+NAME: meta_config_file_hub_read_site_yaml
+#+HEADER: :noweb yes
+#+BEGIN_SRC d
+static template configReadSiteYAML() {
+ import std.file;
+ import std.path;
+ import sisudoc.ocda.meta;
+ import sisudoc.ocda.io_in.paths_source;
+ import sisudoc.ocda.meta.rgx_files;
+ import sisudoc.ocda.meta.rgx;
+ final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) {
+ string _configuration = configReadInSiteYAML!()(_manifested, _env);
+ auto _conf_file_details = configFilePaths!()(_manifested, _env);
+ string _conf_yaml_fn = _conf_file_details.config_filename_site;
+ YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
+ return _yaml_conf;
+ }
+}
+static template configReadDocYAML() {
+ import std.file;
+ import std.path;
+ import sisudoc.ocda.meta;
+ import sisudoc.ocda.io_in.paths_source;
+ final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) {
+ string _configuration = configReadInDocYAML!()(_manifested, _env);
+ auto _conf_file_details = configFilePaths!()(_manifested, _env);
+ string _conf_yaml_fn = _conf_file_details.config_filename_document;
+ YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
+ return _yaml_conf;
+ }
+}
+#+END_SRC
+
+* B. get _markup source_, read file (.readText) [#A]:module:source_files:
+** _module template_
+
+#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d"
+#+HEADER: :noweb yes
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ module source_read_source_files;<BR>
+ - open markup files<BR>
+ - if master file scan for addional files to import/insert
++/
+module sisudoc.ocda.io_in.read_source_files;
+@safe:
+template spineRawMarkupContent() {
+ import std.digest.sha;
+ import std.file;
+ import std.path;
+ import sisudoc.ocda.meta;
+ import sisudoc.ocda.io_in.paths_source;
+ import sisudoc.ocda.meta.rgx_files;
+ import sisudoc.ocda.meta.rgx;
+ mixin spineRgxIn;
+ static auto rgx = RgxI();
+ mixin spineRgxFiles;
+ static auto rgx_files = RgxFiles();
+ string[] _images=[];
+ /+ ↓ the images a markup source shows.
+ matched on the image markup itself ({ name.png }image, and the url and
+ ~^ forms), ensure a filename-shaped word in prose is not mistaken for
+ one, and over every match (so a line showing two images reports two).
+ +/
+ string[] _extract_images(S)(S content_block) {
+ string[] images_;
+ string _content_block = content_block.to!string;
+ foreach (m; _content_block.matchAll(rgx.smid_image)) {
+ images_ ~= m["image"].to!string;
+ }
+ return images_;
+ }
+ auto rawsrc = RawMarkupContent();
+ struct ST_contents_inserts_images {
+ char[][] contents;
+ string[] insert_files;
+ string[] images;
+ }
+ struct ST_header_content_inserts_images {
+ char[] header;
+ char[][] src_txt;
+ string[] insert_files;
+ string[] images;
+ }
+ struct ST_doc_digest {
+ ubyte[32] markup_doc;
+ ubyte[32] header;
+ ubyte[32] text;
+ }
+ struct ST_doc_parts {
+ char[] header_raw;
+ char[][] sourcefile_body_content;
+ string[] insert_file_list;
+ string[] images_list;
+ ST_doc_digest doc_digest;
+ }
+ ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) {
+ ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct
+ = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src);
+ return _0_header_1_body_content_2_insert_filelist_struct;
+ }
+ struct RawMarkupContent {
+ final sourceContent(in string fn_src) {
+ auto raw = MarkupRawUnit();
+ string source_txt_str
+ = raw.markupSourceReadIn(fn_src);
+ return source_txt_str;
+ }
+ final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)(
+ O _opt_action,
+ in string source_txt_str,
+ in string fn_src=""
+ ) {
+ auto raw = MarkupRawUnit();
+ string[] insert_file_list_get;
+ string[] images_list_get;
+ ST_header_content_inserts_images st
+ = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str);
+ char[] header_raw = st.header;
+ char[][] sourcefile_body_content = st.src_txt;
+ if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise
+ auto ins = Inserts();
+ ST_contents_inserts_images _cii
+ = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
+ sourcefile_body_content = _cii.contents;
+ insert_file_list_get = _cii.insert_files.dup;
+ images_list_get = _cii.images.dup;
+ } else if (_opt_action.source_or_pod) {
+ auto ins = Inserts();
+ ST_contents_inserts_images _cii
+ = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
+ images_list_get = _cii.images.dup;
+ } // image_list, if path could take sha256 digests already here?
+ string header_type = "";
+ ST_doc_digest dig;
+ {
+ dig.markup_doc = source_txt_str.sha256Of;
+ dig.header = st.header.sha256Of;
+ dig.text = sourcefile_body_content.sha256Of;
+ }
+ ST_doc_parts ret;
+ {
+ ret.header_raw = st.header;
+ ret.sourcefile_body_content = sourcefile_body_content;
+ ret.insert_file_list = insert_file_list_get;
+ ret.images_list = images_list_get;
+ ret.doc_digest = dig;
+ }
+ return ret;
+ }
+ }
+ struct MarkupRawUnit {
+ import std.digest.sha;
+ import std.file;
+ <<meta_markup_source_raw_read_file_source_string>>
+ <<meta_markup_source_raw_doc_header_and_content_split>>
+ <<meta_markup_source_raw_source_line_array>>
+ <<meta_markup_source_raw_read_in_file>>
+ <<meta_markup_source_raw_struct_of_header_and_body>>
+ <<meta_markup_source_raw_get_insert_source_line_array>>
+ }
+ struct Inserts {
+ struct ST_contents_and_images {
+ char[][] insert_contents;
+ string[] images;
+ }
+ ST_contents_and_images scan_subdoc_source(O)(
+ O _opt_action,
+ char[][] markup_sourcefile_insert_content,
+ string fn_src
+ ) {
+ <<meta_inserts_scan>>
+ foreach (line; markup_sourcefile_insert_content) {
+ <<meta_inserts_scan_loop>>
+ } // end src subdoc (inserts) loop
+ <<meta_inserts_scan_post>>
+ }
+ ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)(
+ O _opt_action,
+ char[][] sourcefile_body_content,
+ string fn_src
+ ) {
+ import std.algorithm;
+ <<meta_master_doc_scan_for_insert_filenames>>
+ foreach (line; sourcefile_body_content) {
+ <<meta_master_doc_scan_for_insert_filenames_loop>>
+ } // end src doc loop
+ <<meta_master_doc_scan_for_insert_filenames_post>>
+ }
+ }
+}
+#+END_SRC
+
+** get markup source, read file :source:markup:
+*** read file, source string [#A] :string:
+
+#+NAME: meta_markup_source_raw_read_file_source_string
+#+BEGIN_SRC d
+final private string readInMarkupSource(in char[] fn_src) {
+ enforce(
+ exists(fn_src) != 0,
+ "file not found: «" ~
+ fn_src ~ "»"
+ );
+ string source_txt_str;
+ try {
+ if (exists(fn_src)) {
+ if (fn_src.getLinkAttributes.attrIsFile) {
+ source_txt_str = fn_src.readText;
+ } else {
+ }
+ }
+ } catch (ErrnoException ex) {
+ } catch (UTFException ex) {
+ // Handle validation errors
+ } catch (FileException ex) {
+ // Handle errors
+ }
+ std.utf.validate(source_txt_str);
+ return source_txt_str;
+}
+#+END_SRC
+
+*** document header & content, array.length == 2 [#A] :array:
+
+here you split document header and body, an array.length == 2
+split is on first match of level A~ (which is required)
+
+#+NAME: meta_markup_source_raw_doc_header_and_content_split
+#+BEGIN_SRC d
+@trusted final private char[][] header0Content1(in string src_text) { // cast(char[])
+ /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/
+ char[][] header_and_content;
+ auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a);
+ header_and_content ~= m.pre;
+ header_and_content ~= m.hit ~ m.post;
+ assert(header_and_content.length == 2,
+ "document markup is broken, header body split == "
+ ~ header_and_content.length.to!string
+ ~ "; (header / body array split should == 2 (split is on level A~))"
+ );
+ return header_and_content;
+}
+#+END_SRC
+
+*** source line array :array:
+
+#+NAME: meta_markup_source_raw_source_line_array
+#+BEGIN_SRC d
+@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[])
+ char[][] source_line_arr
+ = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding);
+ return source_line_arr;
+}
+#+END_SRC
+
+*** source content raw line array :array:
+- used for regular .sst files; master .ssm files and; .ssi inserts
+- regex is passed for relevant enforce match
+**** read in file
+
+#+NAME: meta_markup_source_raw_read_in_file
+#+BEGIN_SRC d
+string markupSourceReadIn(in string fn_src) {
+ static auto rgx_files = RgxFiles();
+ enforce(
+ fn_src.match(rgx_files.src_pth_sst_or_ssm),
+ "not a dr markup filename: «" ~
+ fn_src ~ "»"
+ );
+ string source_txt_str = readInMarkupSource(fn_src);
+ return source_txt_str;
+}
+#+END_SRC
+
+**** tuple (a) header, (b) body content, (c) file insert list & (d) image list?
+
+- header
+- body content
+- file insert list
+- [image list?]
+
+#+NAME: meta_markup_source_raw_struct_of_header_and_body
+#+BEGIN_SRC d
+ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) {
+ string[] file_insert_list = [];
+ string[] images_list = [];
+ char[][] hc = header0Content1(source_txt_str);
+ char[] header = hc[0];
+ char[] source_txt = hc[1];
+ char[][] source_line_arr = markupSourceLineArray(source_txt);
+ ST_header_content_inserts_images ret;
+ {
+ ret.header = header;
+ ret.src_txt = source_line_arr;
+ ret.insert_files = file_insert_list;
+ ret.images = images_list;
+ }
+ return ret;
+}
+#+END_SRC
+
+**** get insert source line array
+
+#+NAME: meta_markup_source_raw_get_insert_source_line_array
+#+BEGIN_SRC d
+final char[][] getInsertMarkupSourceContentRawLineArray(
+ in char[] fn_src_insert,
+ Regex!(char) rgx_file
+) {
+ enforce(
+ fn_src_insert.match(rgx_file),
+ "not a dr markup filename: «" ~
+ fn_src_insert ~ "»"
+ );
+ string source_txt_str = readInMarkupSource(fn_src_insert);
+ char[][] source_line_arr = markupSourceLineArray(source_txt_str);
+ return source_line_arr;
+}
+#+END_SRC
+
+** get markup source, master file & inserts :masterfile:inserts:
+*** scan inserts (sub-document) source :scan_insert_src:
+**** scan subdoc source
+
+#+NAME: meta_inserts_scan
+#+BEGIN_SRC d
+char[][] contents_insert;
+int code_block_status = 0;
+enum codeBlock { off, curly, tic, quotemarks }
+auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
+auto markup_src_file_path = fn_pth_full.captures[1];
+#+END_SRC
+
+**** loop insert (sub-document)
+
+#+NAME: meta_inserts_scan_loop
+#+BEGIN_SRC d
+if (code_block_status == codeBlock.curly) {
+ if (line.matchFirst(rgx.block_curly_code_close)) {
+ code_block_status = codeBlock.off;
+ }
+ contents_insert ~= line;
+} else if (line.matchFirst(rgx.block_curly_code_open)) {
+ code_block_status = codeBlock.curly;
+ contents_insert ~= line;
+} else if (code_block_status == codeBlock.quotemarks) {
+ if (line.matchFirst(rgx.block_quotemarks_close)) {
+ code_block_status = codeBlock.off;
+ }
+ contents_insert ~= line;
+} else if (code_block_status == codeBlock.tic) {
+ if (line.matchFirst(rgx.block_tic_close)) {
+ code_block_status = codeBlock.off;
+ }
+ contents_insert ~= line;
+} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
+ code_block_status = codeBlock.quotemarks;
+ contents_insert ~= line;
+} else if (line.matchFirst(rgx.block_tic_code_open)) {
+ code_block_status = codeBlock.tic;
+ contents_insert ~= line;
+} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
+ auto insert_fn = m.captures[2];
+ auto insert_sub_pth = m.captures[1];
+ auto fn_src_insert
+ = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
+ auto raw = MarkupRawUnit();
+ auto markup_sourcesubfile_insert_content
+ = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
+ debug(insert_file) {
+ writeln(line);
+ writeln(fn_src_insert);
+ writeln(
+ " length contents insert array: ",
+ markup_sourcesubfile_insert_content.length
+ );
+ }
+ if (_opt_action.source_or_pod) {
+ _images ~= _extract_images(markup_sourcesubfile_insert_content);
+ }
+ auto ins = Inserts();
+ /+
+ - 1. load file
+ - 2. read lines
+ - 3. scan lines
+ - a. if filename insert, and insert filename
+ - repeat 1
+ - b. else
+ - add line to new array;
+ - build image list, search for any image files to add to image list
+ +/
+} else {
+ contents_insert ~= line; // images to extract for image list?
+ if (_opt_action.source_or_pod) {
+ string[] _image_linelist = _extract_images(line);
+ if (_image_linelist.length > 0) {
+ _images ~= _image_linelist;
+ }
+ }
+}
+#+END_SRC
+
+**** post loop
+
+#+NAME: meta_inserts_scan_post
+#+BEGIN_SRC d
+ST_contents_and_images ret;
+{
+ ret.insert_contents = contents_insert;
+ ret.images = _images;
+}
+return ret;
+#+END_SRC
+
+*** scan document source :scan_src:
+**** scan doc source
+
+#+NAME: meta_master_doc_scan_for_insert_filenames
+#+BEGIN_SRC d
+char[][] contents;
+int code_block_status = 0;
+enum codeBlock { off, curly, tic, quotemarks }
+auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
+auto markup_src_file_path = fn_pth_full.captures[1];
+char[][] contents_insert;
+string[] _images =[];
+string[] insert_file_list =[];
+#+END_SRC
+
+**** include inserts: _loop master_ scan for inserts (insert documents)
+
+#+NAME: meta_master_doc_scan_for_insert_filenames_loop
+#+BEGIN_SRC d
+if (code_block_status == codeBlock.curly) {
+ if (line.matchFirst(rgx.block_curly_code_close)) {
+ code_block_status = codeBlock.off;
+ }
+ contents ~= line;
+} else if (line.matchFirst(rgx.block_curly_code_open)) {
+ code_block_status = codeBlock.curly;
+ contents ~= line;
+} else if (code_block_status == codeBlock.quotemarks) {
+ if (line.matchFirst(rgx.block_quotemarks_close)) {
+ code_block_status = codeBlock.off;
+ }
+ contents ~= line;
+} else if (code_block_status == codeBlock.tic) {
+ if (line.matchFirst(rgx.block_tic_close)) {
+ code_block_status = codeBlock.off;
+ }
+ contents ~= line;
+} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
+ code_block_status = codeBlock.quotemarks;
+ contents ~= line;
+} else if (line.matchFirst(rgx.block_tic_code_open)) {
+ code_block_status = codeBlock.tic;
+ contents ~= line;
+} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
+ auto insert_fn = m.captures[2];
+ auto insert_sub_pth = m.captures[1];
+ auto fn_src_insert
+ = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
+ insert_file_list ~= fn_src_insert.to!string;
+ auto raw = MarkupRawUnit();
+ /+ TODO +/
+ auto markup_sourcefile_insert_content
+ = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
+ debug(insert_file) {
+ writeln(line);
+ writeln(fn_src_insert);
+ writeln(
+ " length contents insert array: ",
+ markup_sourcefile_insert_content.length
+ );
+ }
+ auto ins = Inserts();
+ ST_contents_and_images contents_insert_st = ins.scan_subdoc_source(
+ _opt_action,
+ markup_sourcefile_insert_content,
+ fn_src_insert.to!string
+ );
+ contents ~= contents_insert_st.insert_contents;
+ if (_opt_action.source_or_pod) {
+ string[] _image_linelist = _extract_images(contents_insert_st.images);
+ if (_image_linelist.length > 0) {
+ _images ~= _image_linelist;
+ }
+ }
+ /+
+ - 1. load file
+ - 2. read lines
+ - 3. scan lines
+ - a. if filename insert, and insert filename
+ - repeat 1
+ - b. else
+ - add line to new array;
+ - build image list, search for any image files to add to image list
+ +/
+} else {
+ contents ~= line;
+ if (_opt_action.source_or_pod) {
+ string[] _image_linelist = _extract_images(line);
+ if (_image_linelist.length > 0) {
+ _images ~= _image_linelist;
+ }
+ }
+}
+#+END_SRC
+
+**** post loop
+
+#+NAME: meta_master_doc_scan_for_insert_filenames_post
+#+BEGIN_SRC d
+string[] images = [];
+foreach(i; uniq(_images.sort())) {
+ images ~= i;
+}
+debug(insert_file) {
+ writeln(__LINE__);
+ writeln(contents.length);
+}
+ST_contents_inserts_images ret;
+{
+ ret.contents = contents;
+ ret.insert_files = insert_file_list;
+ ret.images = images;
+}
+return ret;
+#+END_SRC
+
+* org includes
+** project version
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* __END__
diff --git a/org/in_source_artefact.org b/org/in_source_artefact.org
deleted file mode 100644
index 07c865f..0000000
--- a/org/in_source_artefact.org
+++ /dev/null
@@ -1,392 +0,0 @@
--*- mode: org -*-
-#+TITLE: sisudoc spine (doc_reform) markup source raw
-#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
-#+FILETAGS: :spine:sourcefile:read:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
-
-* A. get doc from artefact
-** _module template_ :module:config_files:
-
-#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d"
-#+HEADER: :noweb yes
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module sisudoc.ocda.abstraction.doc_from_artefact;
-@safe:
-/+ ↓ a document, from an artefact rather than from markup
- spineAbstraction() parses markup and hands the output writers a doc:
- an abstraction and a doc_matters. This does the same from a .ssp or a
- .ocda.db, filling the same docMattersMake() arguments, so that what
- comes out is the value the writers already take and they never learn
- which of the two made it.
- .
- Three of the seven arguments come from the run and are the same either
- way: program_info, opt_action, and the conf half of conf_make_meta,
- which is site and run scoped (urls, papersize, the search database) and
- has no business in a document's own artefact. The other four come from
- the artefact:
- .
- conf_make_meta meta and make from the @meta and @make blocks, over
- the site config the run assembled
- doc_has docHasFromAbstraction, over the objects
- doc_digest the markup digest from @source
- manifest synthesised from the artefact's path and the
- document's own name and language
- .
- What is not here, deliberately: insert_file_list, which is read only by
- source_pod.d for --source and --pod2. No artefact carries the markup,
- so a pod cannot be rebuilt from one, and the empty list is the truthful
- answer rather than a gap.
-+/
-template spineDocFromArtefact() {
- import std.algorithm : endsWith;
- import std.array;
- import std.conv : to;
- import std.path;
- import std.process : thisProcessID;
- import std.regex;
- import std.stdio;
- import std.string;
- import sisudoc.ocda.meta.conf_make_meta_structs;
- import sisudoc.ocda.meta.doc_matters;
- import sisudoc.ocda.io_in.paths_source;
- import sisudoc.ocda.abstraction.doc_has;
- import sisudoc.ocda.abstraction.load;
- import sisudoc.ocda.meta.topic_register;
- mixin spineDocMatters;
- mixin spineDocHasFromAbstraction;
- mixin spineAbstractionLoad;
- mixin spineTopicRegister;
- /+ ↓ a .ssp header key from a field name: the first underscore becomes the
- dot that separates the group, so title_main is title.main and
- rights_copyright_text is rights.copyright_text. The writer's keys are
- formed the same way, which is what lets this be read off the struct
- rather than kept as a second list to fall out of step with it.
- +/
- private string _sspKeyOf(string _member) {
- foreach (i, c; _member) {
- if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; }
- }
- return _member;
- }
- /+ ↓ the document's own header, over the site config the run assembled.
- every string field of MetaComposite is looked for under its own key,
- so a field added there and emitted by the writer arrives here without
- anything being added. The arr fields are split from the string they
- come from, as agreed: that is parsing a value, not deriving a
- structure.
- +/
- ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) {
- ConfComposite _out = _site_conf; // conf: site and run scoped, kept
- static foreach (_m; __traits(allMembers, MetaComposite)) {
- static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{
- enum string _key = _sspKeyOf(_m);
- if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; }
- }}
- }
- /+ ↓ title_sub is a copy of title_subtitle, made where the header is
- read, and is what epub3 puts in dc:title id="subtitle". It is not
- emitted, being the same string twice; it is remade here the way the
- yaml reader makes it +/
- _out.meta.title_sub = _out.meta.title_subtitle;
- /+ ↓ creator.author is the author list joined with ", ", so splitting on
- it gives the list back +/
- if (_out.meta.creator_author.length > 0) {
- foreach (_a; _out.meta.creator_author.split(", ")) {
- if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; }
- }
- }
- /+ ↓ the topic register arrays, by the one rule the yaml reader uses:
- the split is not a plain one and deriving it twice would be two
- answers to the same question +/
- {
- auto _tr = topicRegisterArrays(_out.meta.classify_topic_register);
- _out.meta.classify_topic_register_arr = _tr.arr;
- _out.meta.classify_topic_register_expanded_arr = _tr.expanded;
- }
- /+ ↓ make: the keys are the field names, with no group to separate +/
- if (auto _v = "doc_type" in _ssp.make) {
- _out.make.doc_type = *_v; _out.make_str.doc_type = *_v;
- }
- if (auto _v = "auto_num_top_at_level" in _ssp.make) {
- _out.make.auto_num_top_at_level = *_v;
- }
- if (auto _v = "auto_num_top_lv" in _ssp.make) {
- try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {}
- }
- if (auto _v = "auto_num_depth" in _ssp.make) {
- try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {}
- }
- if (auto _v = "breaks" in _ssp.make) {
- _out.make.breaks = *_v; _out.make_str.breaks = *_v;
- }
- if (auto _v = "home_button_text" in _ssp.make) {
- _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v;
- }
- /+ ↓ footer is a list, joined with the separator the format escapes +/
- if (auto _v = "footer" in _ssp.make) {
- _out.make.footer = sspUnescapeSplit(*_v);
- _out.make_str.footer = _out.make.footer;
- }
- return _out;
- }
- /+ ↓ where a document loaded from an artefact says it is.
- an artefact names its document (% Source:) and its language (@source
- language) but not where the pod holding its images sits, so that is
- taken from where the artefact itself was found:
- .
- pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/
- pod/<uid>.ocda.db -> pod/<doc>/
- .
- which is where --pod2 puts them both. The images beside a .ssp are
- then found where html.d already looks; a .ocda.db carries its own and
- does not need the directory to exist.
- +/
- struct ST_ArtefactPaths {
- string pod_dir;
- string src_file_with_path;
- }
- ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) {
- ST_ArtefactPaths _out;
- string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en";
- string _dir = _artefact.dirName;
- if (_artefact.endsWith(".ssp")) {
- /+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/
- _out.pod_dir = (_dir.baseName == "abstraction")
- ? _dir.dirName.dirName : _dir;
- } else {
- /+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/
- string _uid = _artefact.baseName;
- foreach (_sfx; [".ocda.db", ".db"]) {
- if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; }
- }
- if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; }
- _out.pod_dir = (_dir.chainPath(_uid).array).to!string;
- }
- _out.src_file_with_path = (_out.pod_dir
- .chainPath("media").chainPath("text").chainPath(_lang)
- .chainPath(_ssp.source).array).to!string;
- return _out;
- }
- /+ ↓ a database's images, written where the writers look for them.
- returns the pod directory to use in place of the one beside the
- artefact, or "" when the database carries no images and the pod
- beside it should stand. Each blob is checked against the sha256 the
- database recorded with it: the two were written together, so a
- mismatch means the file is damaged, and saying so is the point of
- having recorded the digest at all.
- +/
- @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) {
- import std.digest : toHexString;
- import std.digest.sha : sha256Of;
- import std.file : mkdirRecurse, write, tempDir;
- import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead;
- mixin spineAbstractionDbRead _dbr;
- auto _files = _dbr.dbReadFiles(_artefact, "image");
- if (_files.length == 0) { return ""; }
- string _root = (tempDir.chainPath("spine-ocda-"
- ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string;
- string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string;
- try {
- _img_dir.mkdirRecurse;
- } catch (Exception ex) {
- stderr.writeln("WARNING: could not make an image directory for ", _artefact,
- ": ", ex.msg);
- return "";
- }
- foreach (_f; _files) {
- string _got = _f.data.sha256Of.toHexString.to!string;
- if (_f.sha256.length > 0 && _got != _f.sha256) {
- stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName,
- " does not match the digest recorded with it (", _f.sha256, " expected, ",
- _got, " found); it is written out as it stands");
- }
- try {
- (_img_dir.chainPath(_f.name).array).to!string.write(_f.data);
- } catch (Exception ex) {
- stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg);
- }
- }
- if (_opt_action.vox_gt_1) {
- writeln(" ", _files.length, " image(s) from the database, in ", _img_dir);
- }
- return _root;
- }
- /+ ↓ the images a .ssp describes, against the ones beside it.
- a .ssp carries no image bytes, so what it can do is say whether the
- files in the pod it sits in are the ones the abstraction was built
- from. Nothing checked that before: a changed or wrong image in the
- directory paired silently.
- +/
- @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) {
- import std.digest : toHexString;
- import std.digest.sha : sha256Of;
- import std.file : exists, read;
- string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string;
- int _checked, _missing, _wrong;
- foreach (_section; _ssp.section_order) {
- foreach (obj; _ssp.abstraction[_section]) {
- foreach (_img; obj.metainfo.sha256.images) {
- if (_img.fileName.length == 0) { continue; }
- string _pth = (_img_dir.chainPath(_img.fileName).array).to!string;
- if (!_pth.exists) { ++_missing; continue; }
- ++_checked;
- string _want = _img.fileHash_sha256.toHexString.to!string;
- string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string;
- if (_want != _got && _want != "00000000000000000000000000000000"
- ~ "00000000000000000000000000000000") {
- ++_wrong;
- stderr.writeln("WARNING: image ", _img.fileName, " beside ",
- _ssp.source, " is not the one the abstraction was built from");
- }
- }
- }
- }
- if (_missing > 0) {
- stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are",
- " not in ", _img_dir);
- }
- if (_opt_action.vox_gt_1 && _checked > 0) {
- writeln(" ", _checked, " image(s) checked against the abstraction's digests",
- (_wrong == 0) ? ", all matching" : "");
- }
- }
- /+ ↓ the doc the output writers take, from an artefact.
- the reading is done here rather than by the caller, and that is not
- only tidiness: main() is @system, so a template mixed in there infers
- @system for everything it declares, and the output writers are @safe
- and will not take objects of such a type. Doing it inside this module,
- which is @safe:, is what makes the loaded objects the same kind of
- thing the parser's are. Called as spineDocFromArtefact!()(...), the
- way spineAbstraction!() is, rather than mixed in.
- +/
- auto spineDocFromArtefact(P,O,E)(
- string _artefact,
- P program_info,
- O _opt_action,
- E _env,
- ConfComposite _site_conf,
- ) {
- auto _loaded = abstractionLoad(_artefact);
- auto _ssp = _loaded.doc;
- auto _pths = artefactPaths(_artefact, _ssp);
- /+ ↓ the images.
- a .ocda.db carries its own, which is what lets it travel on its
- own, so they are taken from it and written where the output writers
- look for images: a directory of this run's making, not the pod
- beside the artefact, which may be somebody's published tree. A .ssp
- only describes its images, so those are the ones in the pod it sits
- in, and are checked against the digests it recorded.
- +/
- string _images_tmp;
- if (_loaded.loaded) {
- if (_artefact.endsWith(".ssp")) {
- _imagesVerify(_pths.pod_dir, _ssp, _opt_action);
- } else {
- string _extracted = _imagesExtract(_artefact, _opt_action);
- if (_extracted.length > 0) {
- /+ ↓ the source path has to move with it: image_dir_path is
- reached from the document's own file, not from the pod +/
- _images_tmp = _extracted;
- _pths.pod_dir = _extracted;
- _pths.src_file_with_path = (_extracted
- .chainPath("media").chainPath("text")
- .chainPath(("language" in _ssp.source_info)
- ? _ssp.source_info["language"] : "en")
- .chainPath(_ssp.source).array).to!string;
- }
- }
- }
- auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path);
- auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf);
- auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action);
- /+ ↓ the digests: the markup digest is what @source carries, and it is
- the one that says which source this abstraction was made from. The
- header and text digests are of parts an artefact does not keep
- separately, and are left unset rather than guessed at
- +/
- ST_ArtefactDigest _dig;
- if (auto _v = "digest" in _ssp.source_info) {
- _dig.markup_doc = _hexToDigest(*_v);
- }
- auto _abstraction = _ssp.abstraction;
- auto _matters = docMattersMake(
- program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init,
- );
- /+ ↓ the same shape spineAbstraction() returns, built the same way: a
- struct of accessors over the locals, with whether the artefact
- could be read at all, which the caller has to know +/
- auto theDOC() {
- struct ST_DOC {
- const auto abstraction() { return _abstraction; }
- auto matters() { return _matters; }
- bool loaded() { return _loaded.loaded; }
- string images_tmp() { return _images_tmp; }
- string note() { return _loaded.note; }
- string source_name() { return abstractionSourceName(_loaded.source); }
- }
- return ST_DOC();
- }
- return theDOC();
- }
- /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from
- an artefact, @source having recorded it; the header and text digests
- are of parts no artefact keeps separately and are left unset rather
- than guessed at. Shaped to match ST_doc_digest, which is declared
- inside spineRawMarkupContent and reachable only by parsing a file.
- +/
- struct ST_ArtefactDigest {
- ubyte[32] markup_doc;
- ubyte[32] header;
- ubyte[32] text;
- }
- private ubyte[32] _hexToDigest(string _hex) {
- ubyte[32] _out;
- if (_hex.length < 64) { return _out; }
- foreach (i; 0..32) {
- try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {}
- }
- return _out;
- }
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/ocda_get.org b/org/ocda_get.org
deleted file mode 100644
index 40effc5..0000000
--- a/org/ocda_get.org
+++ /dev/null
@@ -1,284 +0,0 @@
--*- mode: org -*-
-#+TITLE: sisudoc spine (doc_reform) object-centric document abstraction
-#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
-#+FILETAGS: :spine:abstraction:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
-
-* Get OCD
-
-get abstracted objects for downstream processing
-
-** _module template_
-
-#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_has.d"
-#+HEADER: :noweb yes
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module sisudoc.ocda.abstraction.doc_has;
-@safe:
-/+ ↓ ST_DocHas from a loaded abstraction
-
- the parser builds ST_DocHas as it goes, with the markup in front of it.
- A document loaded from a .ssp or a .ocda.db has to arrive at the same
- value from what the artefact carries, and this is where that is done.
-
- Every one of these is an *index over properties the artefact already
- holds*, not a re-derivation of something it does not:
-
- imagelist the .image records' filenames
- segnames_lv4 .anchor of each body heading at level 4
- segnames_lv_0_to_4 .segment_epub of each heading at level 4 or less
- tag_associations .anchor, .anchor_tag and .segment_* per object
- section_keys_sequenced which sections are non-empty, plus the run's
- own output flags
-
- That distinction is the line the .ssp skill draws: assembling an index
- is fine and cannot drift, because a wrong index would not survive the
- round trip that produced its inputs; recomputing dom_status or ancestors
- would be re-derivation and is exactly what ocda exists to remove. If a
- value is not in the file and cannot be indexed out of what is, it must
- be added to the artefact instead of reconstructed here.
-
- The counts come from the @doc_has block rather than being recounted, for
- the same reason.
-+/
-template spineDocHasFromAbstraction() {
- import std.algorithm : sort, uniq;
- import std.array;
- import std.conv : to;
- import std.json : JSONValue;
- import std.regex;
- import sisudoc.ocda.meta.metadoc_object_setter;
- import sisudoc.ocda.meta.rgx;
- mixin ObjectSetter;
- mixin spineRgxIn;
- /+ ↓ the sections, in the order a document holds them +/
- enum string[] doc_sections = [
- "head", "toc", "body", "endnotes",
- "glossary", "bibliography", "bookindex", "blurb", "tail",
- ];
- /+ ↓ the tail sections that become segments of their own, in the order
- after_doc_determine_segnames appends them +/
- enum string[] tail_sections = [
- "endnotes", "glossary", "bibliography", "bookindex", "blurb",
- ];
- private uint _count(string[string] _doc_has, string _key) {
- if (auto _v = _key in _doc_has) {
- try { return (*_v).to!uint; } catch (Exception ex) { return 0; }
- }
- return 0;
- }
- private bool _has_section(A)(A _abst, string _section) {
- if (auto _s = _section in _abst) { return (*_s).length > 1; }
- return false;
- }
- /+ ↓ which document sections each output format walks, and in what order.
-
- the same rule the parser applies: the three that are always there,
- then each tail section that the document actually has, then "tail"
- for the formats that close with one. It depends on the run as well as
- on the document, since only html and epub take the tail.
- +/
- string[][string] sectionKeysSequenced(A,O)(A _abst, O _opt_action) {
- string[][string] _keys = [
- "scroll": ["head", "toc", "body",],
- "seg": ["head", "toc", "body",],
- "sql": ["head", "body",],
- "latex": ["head", "toc", "body",],
- "text": ["head", "toc", "body",],
- ];
- if (_has_section(_abst, "endnotes")) {
- _keys["scroll"] ~= "endnotes";
- _keys["seg"] ~= "endnotes";
- _keys["latex"] ~= "endnotes";
- _keys["text"] ~= "endnotes";
- }
- if (_has_section(_abst, "glossary")) {
- foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "glossary"; }
- }
- if (_has_section(_abst, "bibliography")) {
- foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bibliography"; }
- }
- if (_has_section(_abst, "bookindex")) {
- foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bookindex"; }
- }
- if (_has_section(_abst, "blurb")) {
- foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "blurb"; }
- }
- if (_opt_action.html_scroll || _opt_action.html_seg || _opt_action.epub) {
- _keys["scroll"] ~= "tail";
- _keys["seg"] ~= "tail";
- }
- return _keys;
- }
- /+ ↓ the whole of it +/
- ST_DocHas docHasFromAbstraction(A,O)(
- A _abst,
- string[string] _doc_has_block,
- O _opt_action,
- ) {
- static auto rgx = RgxI();
- ST_DocHas _out;
- _out.inline_links = _count(_doc_has_block, "inline_links");
- _out.inline_notes_reg = _count(_doc_has_block, "inline_notes_reg");
- _out.inline_notes_star = _count(_doc_has_block, "inline_notes_star");
- _out.codeblocks = _count(_doc_has_block, "codeblocks");
- _out.tables = _count(_doc_has_block, "tables");
- _out.blocks = _count(_doc_has_block, "blocks");
- _out.groups = _count(_doc_has_block, "groups");
- _out.poems = _count(_doc_has_block, "poems");
- _out.quotes = _count(_doc_has_block, "quotes");
- string[] _images;
- string[string][string] _tag_assoc;
- /+ ↓ segnames["html"] is seeded with the toc, which is a segment of its
- own before any level 4 heading opens one +/
- _out.segnames_lv4 ~= "toc";
- foreach (_section; doc_sections) {
- if (_section !in _abst) { continue; }
- /+ ↓ head is always walked; every other section only when it holds
- more than the placeholder object, which is the guard the parser
- puts on each of its section loops
- +/
- if (_section != "head" && !_has_section(_abst, _section)) { continue; }
- foreach (obj; _abst[_section]) {
- /+ ↓ the images this object names, as the abstraction recorded them +/
- foreach (_img; obj.metainfo.sha256.images) { _images ~= _img.fileName; }
- /+ ↓ every object: its html anchor is in a segment, and its epub
- segment anchor stands for itself +/
- if (obj.tags.anchor_tag_html.length > 0) {
- _tag_assoc[obj.tags.anchor_tag_html]["seg_lv4"] = obj.tags.in_segment_html;
- }
- if (obj.tags.segment_anchor_tag_epub.length > 0) {
- _tag_assoc[obj.tags.segment_anchor_tag_epub]["seg_lv1to4"]
- = obj.tags.segment_anchor_tag_epub;
- }
- /+ ↓ a heading's own anchor tags, and the anchors declared inline in
- an object's text, both point at the segment the object is in +/
- /+ ↓ a heading above level 4 opens no html segment of its own, so a
- cross reference to it has to land on the level 4 segment that
- follows it. .segment is exactly that (the segment a heading
- opens or falls into), where .segment_html_is is the segment an
- object sits in; epub segments go down to level 1, so there
- .segment_epub stands for the heading itself
- +/
- if (obj.tags.heading_lev_anchor_tag.length > 0
- && obj.tags.in_segment_html.length > 0
- && obj.metainfo.heading_lev_markup >= 4
- ) {
- _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv4"]
- = obj.tags.in_segment_html;
- if (obj.tags.segment_anchor_tag_epub.length > 0) {
- _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv1to4"]
- = obj.tags.segment_anchor_tag_epub;
- }
- }
- foreach (m; obj.text.matchAll(rgx.inline_link_anchor)) {
- string _a = m["anchor"].to!string;
- if (_a.length == 0 || _a in _tag_assoc) { continue; }
- _tag_assoc[_a]["seg_lv4"] = obj.tags.html_segment_anchor_tag_is;
- _tag_assoc[_a]["seg_lv1to4"] = obj.tags.epub_segment_anchor_tag_is;
- }
- /+ ↓ the segment name lists, in document order +/
- if (obj.metainfo.is_a == "heading") {
- if (obj.metainfo.heading_lev_markup <= 4) {
- _out.segnames_lv_0_to_4 ~= obj.tags.segment_anchor_tag_epub;
- }
- if (_section == "body" && obj.metainfo.heading_lev_markup == 4) {
- _out.segnames_lv4 ~= obj.tags.anchor_tag_html;
- }
- /+ ↓ a heading above level 4 opens no html segment of its own, so
- a cross reference to it lands on the level 4 segment that
- follows. That is .segment_lv4_is, resolved in ocda where the
- finished sections are to hand and carried by the artefact, so
- there is nothing to work out here
- +/
- if (obj.tags.segment_lv4_is.length > 0) {
- /+ ↓ seg_lv4 only, and not through heading_lev_anchor_tag: that
- names the level 4 heading itself, whose own entry says it
- opens that segment rather than merely sits in it. epub
- segments go down to level 1, so seg_lv1to4 is untouched
- +/
- foreach (_k; [obj.metainfo.identifier, obj.tags.anchor_tag_html]) {
- if (_k.length > 0) { _tag_assoc[_k]["seg_lv4"] = obj.tags.segment_lv4_is; }
- }
- }
- }
- }
- }
- foreach (_section; tail_sections) {
- if (_has_section(_abst, _section)) { _out.segnames_lv4 ~= _section; }
- }
- /+ ↓ the body's objects, in a pass of their own after every anchor above
- has been recorded, because the parser does it in that order and the
- order is what decides two of these entries.
- .
- each body object's identifier maps to the segment it is in. seg_lv4
- is set only where it is not already, an anchor of the same name
- having said it *names* a segment rather than sits in one; seg_lv1to4
- is set always. A heading's own anchor can be a bare number taken
- from its text ("2. Shorter Terms" gives the anchor "2"), which then
- collides with the ocn of an unrelated object, and this is the order
- that resolves the collision the way the parser resolves it: in
- free_culture, ocn 2 keeps seg_lv4 "them" from the anchor and takes
- seg_lv1to4 "_part_1" from itself.
- +/
- if (_has_section(_abst, "body")) {
- foreach (obj; _abst["body"]) {
- if (obj.metainfo.identifier.length == 0) { continue; }
- if (!((obj.metainfo.identifier in _tag_assoc)
- && ("seg_lv4" in _tag_assoc[obj.metainfo.identifier]))
- ) {
- _tag_assoc[obj.metainfo.identifier]["seg_lv4"]
- = obj.tags.html_segment_anchor_tag_is;
- }
- _tag_assoc[obj.metainfo.identifier]["seg_lv1to4"]
- = obj.tags.epub_segment_anchor_tag_is;
- }
- }
- _out.imagelist = _images.sort.uniq.array;
- _out.tag_associations = _tag_assoc;
- _out.section_keys_sequenced = sectionKeysSequenced(_abst, _opt_action);
- return _out;
- }
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/out_src_abstraction_ocda_peg_ssp.org b/org/out_ocda_peg_ssp.org
index 511c589..511c589 100644
--- a/org/out_src_abstraction_ocda_peg_ssp.org
+++ b/org/out_ocda_peg_ssp.org
diff --git a/org/out_src_abstraction_sqlite_db.org b/org/out_ocda_sqlite_db.org
index 70c118a..70c118a 100644
--- a/org/out_src_abstraction_sqlite_db.org
+++ b/org/out_ocda_sqlite_db.org