diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-11 21:45:24 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-12 12:39:06 -0400 |
| commit | c6a479b9bd8bdd34ee9699e10247ebdeb183e032 (patch) | |
| tree | 1eab495f7fbf5467c023651aa7f0ee5009215064 /org | |
| parent | spine: a .ocda.db can be fetched, as a pod zip can (diff) | |
org renames
Diffstat (limited to 'org')
| -rw-r--r-- | org/in_abstraction_artefacts.org (renamed from org/in_source_files.org) | 1564 | ||||
| -rw-r--r-- | org/in_markup_source_files.org | 801 | ||||
| -rw-r--r-- | org/in_source_artefact.org | 392 | ||||
| -rw-r--r-- | org/ocda_get.org | 284 | ||||
| -rw-r--r-- | org/out_ocda_peg_ssp.org (renamed from org/out_src_abstraction_ocda_peg_ssp.org) | 0 | ||||
| -rw-r--r-- | org/out_ocda_sqlite_db.org (renamed from org/out_src_abstraction_sqlite_db.org) | 0 |
6 files changed, 1493 insertions, 1548 deletions
diff --git a/org/in_source_files.org b/org/in_abstraction_artefacts.org index 78141b7..d8ce545 100644 --- a/org/in_source_files.org +++ b/org/in_abstraction_artefacts.org @@ -17,768 +17,180 @@ - [[./doc-reform.org][doc-reform.org]] [[./][org/]] -* A. get _config file_, read in (.readText) [#A] -** _module template_ :module:config_files: +* A. load abstraction - sisudoc.ocda.abstraction.load -#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d" +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d" #+HEADER: :noweb yes #+BEGIN_SRC d <<doc_header_including_copyright_and_license>> -/++ - read configuration files<BR> - - read config files<BR> - meta_config_files.d -+/ -module sisudoc.ocda.io_in.read_config_files; +module sisudoc.ocda.abstraction.load; @safe: -import std.file; -import std.path; -import sisudoc.ocda.meta; -import sisudoc.ocda.io_in.paths_source; -import sisudoc.ocda.meta.rgx_files; -import sisudoc.ocda.meta.rgx; -<<meta_config_file_hub_read_site_config>> -<<meta_config_file_hub_read_document_config>> -<<meta_config_file_hub_read_site_yaml>> -#+END_SRC - -*** read config files (config local site & dr document make) (yaml) -**** site configuration - -SEE NOTES on configuration hierarchy in spine.org - -#+NAME: meta_config_file_hub_read_site_config -#+BEGIN_SRC d -template readConfigSite() { - @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) { - mixin spineRgxIn; - static auto rgx = RgxI(); - string conf_filename = "NONE"; - string config_file_str; - string default_config_file_str = format(q"┃ -flag: - act0: "--html" - act1: "--html --epub" -output: - path: "%s" -default: - language: "en" - papersize: "a4" - text_wrap: "80" - digest: "sha256" -webserv: - http: "%s" - host: "%s" - data_http: "%s" - data_host: "%s" - data_root_url: "%s" - data_root_path: "%s" - data_root_part: "" - images_root_part: "image" - cgi_search_form_title: "%s" - cgi_http: "%s" - cgi_host: "%s" - cgi_bin_url: "%s" - cgi_bin_subpath: "%s" - cgi_bin_path: "%s" - cgi_search_script: "%s" - cgi_port: "" - cgi_user: "" - cgi_action: "%s" - db_sqlite_path: "%s" - db_sqlite_filename: "%s" - db_pg_table: "" - db_pg_user: "" -┃", - _cfg.processing_path_doc_root, // doc root - _cfg.http_request_type, // http - _cfg.http_host, // host / domain - _cfg.http_request_type, // data "http" or "https" - _cfg.http_host, // data domain "localhost" - _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org" - _cfg.processing_path_doc_root, // data root path - _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search" - _cfg.http_request_type, // cgi http - _cfg.http_host, // cgi host - _cfg.cgi_url_root, // cgi bin url - _cfg.cgi_bin_subpath, // cgi bin path - _cfg.cgi_bin_root, // cgi bin path - _cfg.cgi_filename, // cgi filename - _cfg.cgi_url_action, // cgi action - _cfg.db_sqlite_path, // sqlite db path - _cfg.db_sqlite_filename, // sqlite db filename -); - foreach(conf_fn; [_conf_file_details.config_filename_site]) { - foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) { - char[] conf_file; - conf_filename = conf_fn; - if (exists(pth)) { - auto f_attrib = pth.getLinkAttributes; - if ( - _conf_file_details.possible_config_path_locations.config_local_site.length == 1 - && f_attrib.attrIsFile - ) { - conf_file = pth.to!(char[]); - conf_filename = pth.baseName; - } else if (f_attrib.attrIsDir) { - conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; - conf_filename = conf_fn; - } - try { - if (exists(conf_file)) { - if (conf_file.getLinkAttributes.attrIsFile) { - if (_opt_action.vox_gt_2 || _opt_action.debug_do) { - writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)"); - } - config_file_str = conf_file.readText; - break; - } - } - } catch (ErrnoException ex) { - } catch (FileException ex) { - } - } - } - if (config_file_str.length > 0) { break; } - } - if (config_file_str.length > 0) { - import dyaml; - Node yaml_root; - try { - yaml_root = Loader.fromString(config_file_str).load(); - } catch (Throwable) { - import std.stdio; - writeln("ERROR failed to read config file content, not parsed as yaml, program default used"); - conf_filename = "VIRTUAL"; - config_file_str = default_config_file_str; - } - } - if (config_file_str.length == 0) { /+ use dummy default config file +/ - // writeln("WARNING config file NOT found, default provided"); - conf_filename = "VIRTUAL"; - config_file_str = default_config_file_str; - } - struct _ConfContent { - string filename() { - return conf_filename; - } - string filetype() { - string _ft = ""; - if (content.match(rgx.yaml_config)) { - _ft = "yaml"; - } - return _ft; - } - string content() { - return config_file_str; - } - } - return _ConfContent(); - } -} -#+END_SRC - -**** document make/config +/+ ↓ one way in, whatever the document is being read from + spine's pipeline is markup -> abstraction -> output. once the abstraction + is serialised, there is more than one thing an abstraction can be read + from, and a consumer should not have to know which it was handed: + . + .sst / .ssm + images the markup source + pod (dir) + images the same, bundled + pod .zip the same, zipped + .ssp + images the abstraction, as text + .ocda.db the abstraction, sqlite, images inside + . + A source may also be remote. A URL ending in .zip or in .ocda.db is + downloaded to a temp file (downloadSourceUrl in + ocda/io_in/read_zip_pod.d, guarded by rgx_url_source and by + --allow-downloads) and the local path is processed in its place. -#+NAME: meta_config_file_hub_read_document_config -#+HEADER: :noweb yes -#+BEGIN_SRC d -static template readConfigDoc() { + Those two travel because each is self-sufficient: a pod holds its + source and images, a .ocda.db holds its abstraction and images. A .ssp + does not, describing its images by name and digest without carrying + them, so it would arrive without them and is not fetched. + . + This module names those sources, tells them apart, and loads the two + that are self-describing artefacts, returning the value the parser + produces. + . + The three source forms are deliberately *not* loaded here. They are the + parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs + the environment, the options and the configuration that spine.d + assembles, none of which belongs in a loader. What this module gives + that case is the dispatch and a plain statement of where to go. ++/ +template spineAbstractionLoad() { + import std.algorithm : endsWith; + import std.conv : to; import std.file; import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.meta.rgx_files; - import sisudoc.ocda.meta.rgx; - @system final auto readConfigDoc(M,E)(M _manifested, E _env) { - mixin spineRgxIn; - static auto rgx = RgxI(); - mixin spineRgxFiles; - static auto rgx_files = RgxFiles(); - string config_file_str; - string conf_filename = "NONE"; - auto _conf_file_details = configFilePaths!()(_manifested, _env); - string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; - foreach(conf_fn; [_conf_file_details.config_filename_document]) { - foreach(pth; possible_config_path_locations) { - char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; - conf_filename = conf_fn; - if (config_file_str.length > 0) { - break; - } - try { - if (exists(conf_file)) { - if (conf_file.getLinkAttributes.attrIsFile) { - config_file_str = conf_file.readText; - break; - } - } - } catch (ErrnoException ex) { - } catch (FileException ex) { - } - } - if (config_file_str.length > 0) { break; } - } - struct _ConfContent { - string filename() { - return conf_filename; - } - string content() { - return config_file_str; - } - string filetype() { - string _ft = ""; - if (content.match(rgx.yaml_config)) { - _ft = "yaml"; - } - return _ft; - } - } - return _ConfContent(); - } -} -#+END_SRC - -*** YAML config (config local site & dr document make) :file:config:hub: + import std.stdio; + import std.string; + import sisudoc.ocda.abstraction.db_in; + mixin spineAbstractionDbRead; // brings the .ssp reader with it -#+NAME: meta_config_file_hub_read_site_yaml -#+HEADER: :noweb yes -#+BEGIN_SRC d -static template configReadSiteYAML() { - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.meta.rgx_files; - import sisudoc.ocda.meta.rgx; - final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) { - string _configuration = configReadInSiteYAML!()(_manifested, _env); - auto _conf_file_details = configFilePaths!()(_manifested, _env); - string _conf_yaml_fn = _conf_file_details.config_filename_site; - YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); - return _yaml_conf; - } -} -static template configReadDocYAML() { - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) { - string _configuration = configReadInDocYAML!()(_manifested, _env); - auto _conf_file_details = configFilePaths!()(_manifested, _env); - string _conf_yaml_fn = _conf_file_details.config_filename_document; - YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); - return _yaml_conf; + enum AbstractionSource { + unknown, + markup, // .sst or .ssm, with its images beside it + pod, // a directory holding pod.manifest + pod_zip, // that directory, zipped + ssp, // .ssp, the abstraction as text, images beside it + ocda_db, // .ocda.db, the abstraction as sqlite, images inside it } -} -#+END_SRC -* B. get _markup source_, read file (.readText) [#A]:module:source_files: -** _module template_ - -#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -/++ - module source_read_source_files;<BR> - - open markup files<BR> - - if master file scan for addional files to import/insert -+/ -module sisudoc.ocda.io_in.read_source_files; -@safe: -template spineRawMarkupContent() { - import std.digest.sha; - import std.file; - import std.path; - import sisudoc.ocda.meta; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.meta.rgx_files; - import sisudoc.ocda.meta.rgx; - mixin spineRgxIn; - static auto rgx = RgxI(); - mixin spineRgxFiles; - static auto rgx_files = RgxFiles(); - string[] _images=[]; - /+ ↓ the images a markup source shows. - matched on the image markup itself ({ name.png }image, and the url and - ~^ forms), ensure a filename-shaped word in prose is not mistaken for - one, and over every match (so a line showing two images reports two). - +/ - string[] _extract_images(S)(S content_block) { - string[] images_; - string _content_block = content_block.to!string; - foreach (m; _content_block.matchAll(rgx.smid_image)) { - images_ ~= m["image"].to!string; - } - return images_; - } - auto rawsrc = RawMarkupContent(); - struct ST_contents_inserts_images { - char[][] contents; - string[] insert_files; - string[] images; - } - struct ST_header_content_inserts_images { - char[] header; - char[][] src_txt; - string[] insert_files; - string[] images; - } - struct ST_doc_digest { - ubyte[32] markup_doc; - ubyte[32] header; - ubyte[32] text; - } - struct ST_doc_parts { - char[] header_raw; - char[][] sourcefile_body_content; - string[] insert_file_list; - string[] images_list; - ST_doc_digest doc_digest; - } - ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) { - ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct - = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src); - return _0_header_1_body_content_2_insert_filelist_struct; - } - struct RawMarkupContent { - final sourceContent(in string fn_src) { - auto raw = MarkupRawUnit(); - string source_txt_str - = raw.markupSourceReadIn(fn_src); - return source_txt_str; - } - final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)( - O _opt_action, - in string source_txt_str, - in string fn_src="" - ) { - auto raw = MarkupRawUnit(); - string[] insert_file_list_get; - string[] images_list_get; - ST_header_content_inserts_images st - = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str); - char[] header_raw = st.header; - char[][] sourcefile_body_content = st.src_txt; - if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise - auto ins = Inserts(); - ST_contents_inserts_images _cii - = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); - sourcefile_body_content = _cii.contents; - insert_file_list_get = _cii.insert_files.dup; - images_list_get = _cii.images.dup; - } else if (_opt_action.source_or_pod) { - auto ins = Inserts(); - ST_contents_inserts_images _cii - = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); - images_list_get = _cii.images.dup; - } // image_list, if path could take sha256 digests already here? - string header_type = ""; - ST_doc_digest dig; - { - dig.markup_doc = source_txt_str.sha256Of; - dig.header = st.header.sha256Of; - dig.text = sourcefile_body_content.sha256Of; - } - ST_doc_parts ret; - { - ret.header_raw = st.header; - ret.sourcefile_body_content = sourcefile_body_content; - ret.insert_file_list = insert_file_list_get; - ret.images_list = images_list_get; - ret.doc_digest = dig; - } - return ret; - } - } - struct MarkupRawUnit { - import std.digest.sha; - import std.file; - <<meta_markup_source_raw_read_file_source_string>> - <<meta_markup_source_raw_doc_header_and_content_split>> - <<meta_markup_source_raw_source_line_array>> - <<meta_markup_source_raw_read_in_file>> - <<meta_markup_source_raw_struct_of_header_and_body>> - <<meta_markup_source_raw_get_insert_source_line_array>> - } - struct Inserts { - struct ST_contents_and_images { - char[][] insert_contents; - string[] images; - } - ST_contents_and_images scan_subdoc_source(O)( - O _opt_action, - char[][] markup_sourcefile_insert_content, - string fn_src - ) { - <<meta_inserts_scan>> - foreach (line; markup_sourcefile_insert_content) { - <<meta_inserts_scan_loop>> - } // end src subdoc (inserts) loop - <<meta_inserts_scan_post>> - } - ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)( - O _opt_action, - char[][] sourcefile_body_content, - string fn_src - ) { - import std.algorithm; - <<meta_master_doc_scan_for_insert_filenames>> - foreach (line; sourcefile_body_content) { - <<meta_master_doc_scan_for_insert_filenames_loop>> - } // end src doc loop - <<meta_master_doc_scan_for_insert_filenames_post>> + string abstractionSourceName(AbstractionSource _s) { + final switch (_s) { + case AbstractionSource.unknown: return "unknown"; + case AbstractionSource.markup: return "markup source (.sst/.ssm)"; + case AbstractionSource.pod: return "pod directory"; + case AbstractionSource.pod_zip: return "pod zip"; + case AbstractionSource.ssp: return ".ssp (abstraction as text)"; + case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)"; } } -} -#+END_SRC -** get markup source, read file :source:markup: -*** read file, source string [#A] :string: - -#+NAME: meta_markup_source_raw_read_file_source_string -#+BEGIN_SRC d -final private string readInMarkupSource(in char[] fn_src) { - enforce( - exists(fn_src) != 0, - "file not found: «" ~ - fn_src ~ "»" - ); - string source_txt_str; - try { - if (exists(fn_src)) { - if (fn_src.getLinkAttributes.attrIsFile) { - source_txt_str = fn_src.readText; - } else { - } + /+ ↓ what is this? by name, and for a directory by what it holds +/ + AbstractionSource abstractionSourceOf(string _path) { + if (_path.length == 0) { return AbstractionSource.unknown; } + if (_path.isValidPath && _path.exists && _path.isDir) { + return (_path.chainPath("pod.manifest").array.exists) + ? AbstractionSource.pod : AbstractionSource.unknown; } - } catch (ErrnoException ex) { - } catch (UTFException ex) { - // Handle validation errors - } catch (FileException ex) { - // Handle errors + if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } + if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } + if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } + if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } + if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } + return AbstractionSource.unknown; } - std.utf.validate(source_txt_str); - return source_txt_str; -} -#+END_SRC - -*** document header & content, array.length == 2 [#A] :array: - -here you split document header and body, an array.length == 2 -split is on first match of level A~ (which is required) - -#+NAME: meta_markup_source_raw_doc_header_and_content_split -#+BEGIN_SRC d -@trusted final private char[][] header0Content1(in string src_text) { // cast(char[]) - /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ - char[][] header_and_content; - auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a); - header_and_content ~= m.pre; - header_and_content ~= m.hit ~ m.post; - assert(header_and_content.length == 2, - "document markup is broken, header body split == " - ~ header_and_content.length.to!string - ~ "; (header / body array split should == 2 (split is on level A~))" - ); - return header_and_content; -} -#+END_SRC - -*** source line array :array: -#+NAME: meta_markup_source_raw_source_line_array -#+BEGIN_SRC d -@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[]) - char[][] source_line_arr - = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding); - return source_line_arr; -} -#+END_SRC - -*** source content raw line array :array: -- used for regular .sst files; master .ssm files and; .ssi inserts -- regex is passed for relevant enforce match - -**** read in file - -#+NAME: meta_markup_source_raw_read_in_file -#+BEGIN_SRC d -string markupSourceReadIn(in string fn_src) { - static auto rgx_files = RgxFiles(); - enforce( - fn_src.match(rgx_files.src_pth_sst_or_ssm), - "not a dr markup filename: «" ~ - fn_src ~ "»" - ); - string source_txt_str = readInMarkupSource(fn_src); - return source_txt_str; -} -#+END_SRC - -**** tuple (a) header, (b) body content, (c) file insert list & (d) image list? - -- header -- body content -- file insert list -- [image list?] - -#+NAME: meta_markup_source_raw_struct_of_header_and_body -#+BEGIN_SRC d -ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) { - string[] file_insert_list = []; - string[] images_list = []; - char[][] hc = header0Content1(source_txt_str); - char[] header = hc[0]; - char[] source_txt = hc[1]; - char[][] source_line_arr = markupSourceLineArray(source_txt); - ST_header_content_inserts_images ret; - { - ret.header = header; - ret.src_txt = source_line_arr; - ret.insert_files = file_insert_list; - ret.images = images_list; + struct LoadedAbstraction { + AbstractionSource source; + string path; + bool loaded; // is .doc filled + string note; // why not, when it is not + SSPdocument doc; } - return ret; -} -#+END_SRC - -**** get insert source line array - -#+NAME: meta_markup_source_raw_get_insert_source_line_array -#+BEGIN_SRC d -final char[][] getInsertMarkupSourceContentRawLineArray( - in char[] fn_src_insert, - Regex!(char) rgx_file -) { - enforce( - fn_src_insert.match(rgx_file), - "not a dr markup filename: «" ~ - fn_src_insert ~ "»" - ); - string source_txt_str = readInMarkupSource(fn_src_insert); - char[][] source_line_arr = markupSourceLineArray(source_txt_str); - return source_line_arr; -} -#+END_SRC - -** get markup source, master file & inserts :masterfile:inserts: - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -*** scan inserts (sub-document) source :scan_insert_src: -**** scan subdoc source - -#+NAME: meta_inserts_scan -#+BEGIN_SRC d -char[][] contents_insert; -int code_block_status = 0; -enum codeBlock { off, curly, tic, quotemarks } -auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); -auto markup_src_file_path = fn_pth_full.captures[1]; -#+END_SRC -**** loop insert (sub-document) + /+ ↓ load what can be loaded from the path alone. -#+NAME: meta_inserts_scan_loop -#+BEGIN_SRC d -if (code_block_status == codeBlock.curly) { - if (line.matchFirst(rgx.block_curly_code_close)) { - code_block_status = codeBlock.off; - } - contents_insert ~= line; -} else if (line.matchFirst(rgx.block_curly_code_open)) { - code_block_status = codeBlock.curly; - contents_insert ~= line; -} else if (code_block_status == codeBlock.quotemarks) { - if (line.matchFirst(rgx.block_quotemarks_close)) { - code_block_status = codeBlock.off; - } - contents_insert ~= line; -} else if (code_block_status == codeBlock.tic) { - if (line.matchFirst(rgx.block_tic_close)) { - code_block_status = codeBlock.off; - } - contents_insert ~= line; -} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { - code_block_status = codeBlock.quotemarks; - contents_insert ~= line; -} else if (line.matchFirst(rgx.block_tic_code_open)) { - code_block_status = codeBlock.tic; - contents_insert ~= line; -} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { - auto insert_fn = m.captures[2]; - auto insert_sub_pth = m.captures[1]; - auto fn_src_insert - = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; - auto raw = MarkupRawUnit(); - auto markup_sourcesubfile_insert_content - = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); - debug(insert_file) { - writeln(line); - writeln(fn_src_insert); - writeln( - " length contents insert array: ", - markup_sourcesubfile_insert_content.length - ); - } - if (_opt_action.source_or_pod) { - _images ~= _extract_images(markup_sourcesubfile_insert_content); - } - auto ins = Inserts(); - /+ - - 1. load file - - 2. read lines - - 3. scan lines - - a. if filename insert, and insert filename - - repeat 1 - - b. else - - add line to new array; - - build image list, search for any image files to add to image list + .ssp and .ocda.db come back filled. the three source forms come back + with loaded = false and a note saying where they are handled, because + reading them needs the manifest, environment and configuration that + spine.d builds, not a file path. +/ -} else { - contents_insert ~= line; // images to extract for image list? - if (_opt_action.source_or_pod) { - string[] _image_linelist = _extract_images(line); - if (_image_linelist.length > 0) { - _images ~= _image_linelist; + LoadedAbstraction abstractionLoad(string _path) { + LoadedAbstraction _out; + _out.path = _path; + _out.source = abstractionSourceOf(_path); + if (!_path.exists) { + _out.note = "no such file or directory"; + return _out; + } + final switch (_out.source) { + case AbstractionSource.ssp: + _out.doc = sspReadFile(_path); + _out.loaded = (_out.doc.section_order.length > 0); + if (!_out.loaded) { _out.note = "no object sections found"; } + break; + case AbstractionSource.ocda_db: + _out.doc = dbReadFile(_path); + _out.loaded = (_out.doc.section_order.length > 0); + if (!_out.loaded) { _out.note = "no object sections found"; } + break; + case AbstractionSource.markup: + case AbstractionSource.pod: + case AbstractionSource.pod_zip: + _out.note = "a source form: read by the parser," + ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the" + ~ " manifest, environment and configuration spine assembles"; + break; + case AbstractionSource.unknown: + _out.note = "not a document source spine knows"; + break; } + return _out; } -} -#+END_SRC - -**** post loop - -#+NAME: meta_inserts_scan_post -#+BEGIN_SRC d -ST_contents_and_images ret; -{ - ret.insert_contents = contents_insert; - ret.images = _images; -} -return ret; -#+END_SRC - -*** scan document source :scan_src: -**** scan doc source -#+NAME: meta_master_doc_scan_for_insert_filenames -#+BEGIN_SRC d -char[][] contents; -int code_block_status = 0; -enum codeBlock { off, curly, tic, quotemarks } -auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); -auto markup_src_file_path = fn_pth_full.captures[1]; -char[][] contents_insert; -string[] _images =[]; -string[] insert_file_list =[]; -#+END_SRC - -**** include inserts: _loop master_ scan for inserts (insert documents) - -#+NAME: meta_master_doc_scan_for_insert_filenames_loop -#+BEGIN_SRC d -if (code_block_status == codeBlock.curly) { - if (line.matchFirst(rgx.block_curly_code_close)) { - code_block_status = codeBlock.off; - } - contents ~= line; -} else if (line.matchFirst(rgx.block_curly_code_open)) { - code_block_status = codeBlock.curly; - contents ~= line; -} else if (code_block_status == codeBlock.quotemarks) { - if (line.matchFirst(rgx.block_quotemarks_close)) { - code_block_status = codeBlock.off; - } - contents ~= line; -} else if (code_block_status == codeBlock.tic) { - if (line.matchFirst(rgx.block_tic_close)) { - code_block_status = codeBlock.off; - } - contents ~= line; -} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { - code_block_status = codeBlock.quotemarks; - contents ~= line; -} else if (line.matchFirst(rgx.block_tic_code_open)) { - code_block_status = codeBlock.tic; - contents ~= line; -} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { - auto insert_fn = m.captures[2]; - auto insert_sub_pth = m.captures[1]; - auto fn_src_insert - = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; - insert_file_list ~= fn_src_insert.to!string; - auto raw = MarkupRawUnit(); - /+ TODO +/ - auto markup_sourcefile_insert_content - = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); - debug(insert_file) { - writeln(line); - writeln(fn_src_insert); - writeln( - " length contents insert array: ", - markup_sourcefile_insert_content.length - ); - } - auto ins = Inserts(); - ST_contents_and_images contents_insert_st = ins.scan_subdoc_source( - _opt_action, - markup_sourcefile_insert_content, - fn_src_insert.to!string - ); - contents ~= contents_insert_st.insert_contents; - if (_opt_action.source_or_pod) { - string[] _image_linelist = _extract_images(contents_insert_st.images); - if (_image_linelist.length > 0) { - _images ~= _image_linelist; + /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a + given artefact really does hold what it should +/ + string[] abstractionLoadSummary(LoadedAbstraction _l) { + string[] _out; + _out ~= "source: " ~ abstractionSourceName(_l.source); + _out ~= "path: " ~ _l.path; + if (!_l.loaded) { + _out ~= "loaded: no"; + if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; } + return _out; } - } - /+ - - 1. load file - - 2. read lines - - 3. scan lines - - a. if filename insert, and insert filename - - repeat 1 - - b. else - - add line to new array; - - build image list, search for any image files to add to image list - +/ -} else { - contents ~= line; - if (_opt_action.source_or_pod) { - string[] _image_linelist = _extract_images(line); - if (_image_linelist.length > 0) { - _images ~= _image_linelist; + _out ~= "loaded: yes"; + if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; } + if ("title.main" in _l.doc.meta) { + _out ~= "title: " ~ _l.doc.meta["title.main"]; + } + if ("creator.author" in _l.doc.meta) { + _out ~= "author: " ~ _l.doc.meta["creator.author"]; + } + _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, " + ~ _l.doc.make.length.to!string ~ " make, " + ~ _l.doc.doc_has.length.to!string ~ " doc_has"; + int _objs, _citable, _headings; + string[] _sections; + foreach (section; _l.doc.section_order) { + int _n; + foreach (obj; _l.doc.abstraction[section]) { + ++_objs; ++_n; + if (obj.metainfo.ocn > 0) { ++_citable; } + if (obj.metainfo.is_a == "heading") { ++_headings; } + } + _sections ~= section ~ " " ~ _n.to!string; } + _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string + ~ " citable, " ~ _headings.to!string ~ " headings)"; + _out ~= "sections: " ~ _sections.join(", "); + return _out; } } #+END_SRC -**** post loop - -#+NAME: meta_master_doc_scan_for_insert_filenames_post -#+BEGIN_SRC d -string[] images = []; -foreach(i; uniq(_images.sort())) { - images ~= i; -} -debug(insert_file) { - writeln(__LINE__); - writeln(contents.length); -} -ST_contents_inserts_images ret; -{ - ret.contents = contents; - ret.insert_files = insert_file_list; - ret.images = images; -} -return ret; -#+END_SRC - -* C. read a .ssp file back into the document abstraction -** _module template_ :module:ssp_files: +* B. read a .ssp file back into the document abstraction - sisudoc.ocda.abstraction.ssp_in #+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp_in.d" #+HEADER: :noweb yes @@ -1154,8 +566,7 @@ template spineAbstractionRead() { } #+END_SRC -* D. read a .db file into the document abstraction -** _module template_ :module:db_files: +* C. read a .db file into the document abstraction - sisudoc.ocda.abstraction.db_in #+HEADER: :tangle "../src/sisudoc/ocda/abstraction/db_in.d" #+HEADER: :noweb yes @@ -1454,175 +865,584 @@ template spineAbstractionDbRead() { } #+END_SRC -* E. read a .db file into the document abstraction -** _module template_ :module:db_files: +* D. get doc from artefact - sisudoc.ocda.abstraction.doc_from_artefact -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d" +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d" #+HEADER: :noweb yes #+BEGIN_SRC d <<doc_header_including_copyright_and_license>> -module sisudoc.ocda.abstraction.load; +module sisudoc.ocda.abstraction.doc_from_artefact; @safe: -/+ ↓ one way in, whatever the document is being read from - spine's pipeline is markup -> abstraction -> output. once the abstraction - is serialised, there is more than one thing an abstraction can be read - from, and a consumer should not have to know which it was handed: +/+ ↓ a document, from an artefact rather than from markup + spineAbstraction() parses markup and hands the output writers a doc: + an abstraction and a doc_matters. This does the same from a .ssp or a + .ocda.db, filling the same docMattersMake() arguments, so that what + comes out is the value the writers already take and they never learn + which of the two made it. . - .sst / .ssm + images the markup source - pod (dir) + images the same, bundled - pod .zip the same, zipped - .ssp + images the abstraction, as text - .ocda.db the abstraction, sqlite, images inside - . - A source may also be remote. A URL ending in .zip or in .ocda.db is - downloaded to a temp file (downloadSourceUrl in - ocda/io_in/read_zip_pod.d, guarded by rgx_url_source and by - --allow-downloads) and the local path is processed in its place. - - Those two travel because each is self-sufficient: a pod holds its - source and images, a .ocda.db holds its abstraction and images. A .ssp - does not, describing its images by name and digest without carrying - them, so it would arrive without them and is not fetched. + Three of the seven arguments come from the run and are the same either + way: program_info, opt_action, and the conf half of conf_make_meta, + which is site and run scoped (urls, papersize, the search database) and + has no business in a document's own artefact. The other four come from + the artefact: . - This module names those sources, tells them apart, and loads the two - that are self-describing artefacts, returning the value the parser - produces. + conf_make_meta meta and make from the @meta and @make blocks, over + the site config the run assembled + doc_has docHasFromAbstraction, over the objects + doc_digest the markup digest from @source + manifest synthesised from the artefact's path and the + document's own name and language . - The three source forms are deliberately *not* loaded here. They are the - parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs - the environment, the options and the configuration that spine.d - assembles, none of which belongs in a loader. What this module gives - that case is the dispatch and a plain statement of where to go. + What is not here, deliberately: insert_file_list, which is read only by + source_pod.d for --source and --pod2. No artefact carries the markup, + so a pod cannot be rebuilt from one, and the empty list is the truthful + answer rather than a gap. +/ -template spineAbstractionLoad() { +template spineDocFromArtefact() { import std.algorithm : endsWith; + import std.array; import std.conv : to; - import std.file; import std.path; + import std.process : thisProcessID; + import std.regex; import std.stdio; import std.string; - import sisudoc.ocda.abstraction.db_in; - mixin spineAbstractionDbRead; // brings the .ssp reader with it - - enum AbstractionSource { - unknown, - markup, // .sst or .ssm, with its images beside it - pod, // a directory holding pod.manifest - pod_zip, // that directory, zipped - ssp, // .ssp, the abstraction as text, images beside it - ocda_db, // .ocda.db, the abstraction as sqlite, images inside it + import sisudoc.ocda.meta.conf_make_meta_structs; + import sisudoc.ocda.meta.doc_matters; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.abstraction.doc_has; + import sisudoc.ocda.abstraction.load; + import sisudoc.ocda.meta.topic_register; + mixin spineDocMatters; + mixin spineDocHasFromAbstraction; + mixin spineAbstractionLoad; + mixin spineTopicRegister; + /+ ↓ a .ssp header key from a field name: the first underscore becomes the + dot that separates the group, so title_main is title.main and + rights_copyright_text is rights.copyright_text. The writer's keys are + formed the same way, which is what lets this be read off the struct + rather than kept as a second list to fall out of step with it. + +/ + private string _sspKeyOf(string _member) { + foreach (i, c; _member) { + if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; } + } + return _member; } - - string abstractionSourceName(AbstractionSource _s) { - final switch (_s) { - case AbstractionSource.unknown: return "unknown"; - case AbstractionSource.markup: return "markup source (.sst/.ssm)"; - case AbstractionSource.pod: return "pod directory"; - case AbstractionSource.pod_zip: return "pod zip"; - case AbstractionSource.ssp: return ".ssp (abstraction as text)"; - case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)"; + /+ ↓ the document's own header, over the site config the run assembled. + every string field of MetaComposite is looked for under its own key, + so a field added there and emitted by the writer arrives here without + anything being added. The arr fields are split from the string they + come from, as agreed: that is parsing a value, not deriving a + structure. + +/ + ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) { + ConfComposite _out = _site_conf; // conf: site and run scoped, kept + static foreach (_m; __traits(allMembers, MetaComposite)) { + static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{ + enum string _key = _sspKeyOf(_m); + if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; } + }} + } + /+ ↓ title_sub is a copy of title_subtitle, made where the header is + read, and is what epub3 puts in dc:title id="subtitle". It is not + emitted, being the same string twice; it is remade here the way the + yaml reader makes it +/ + _out.meta.title_sub = _out.meta.title_subtitle; + /+ ↓ creator.author is the author list joined with ", ", so splitting on + it gives the list back +/ + if (_out.meta.creator_author.length > 0) { + foreach (_a; _out.meta.creator_author.split(", ")) { + if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; } + } + } + /+ ↓ the topic register arrays, by the one rule the yaml reader uses: + the split is not a plain one and deriving it twice would be two + answers to the same question +/ + { + auto _tr = topicRegisterArrays(_out.meta.classify_topic_register); + _out.meta.classify_topic_register_arr = _tr.arr; + _out.meta.classify_topic_register_expanded_arr = _tr.expanded; + } + /+ ↓ make: the keys are the field names, with no group to separate +/ + if (auto _v = "doc_type" in _ssp.make) { + _out.make.doc_type = *_v; _out.make_str.doc_type = *_v; + } + if (auto _v = "auto_num_top_at_level" in _ssp.make) { + _out.make.auto_num_top_at_level = *_v; + } + if (auto _v = "auto_num_top_lv" in _ssp.make) { + try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {} + } + if (auto _v = "auto_num_depth" in _ssp.make) { + try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {} + } + if (auto _v = "breaks" in _ssp.make) { + _out.make.breaks = *_v; _out.make_str.breaks = *_v; + } + if (auto _v = "home_button_text" in _ssp.make) { + _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v; + } + /+ ↓ footer is a list, joined with the separator the format escapes +/ + if (auto _v = "footer" in _ssp.make) { + _out.make.footer = sspUnescapeSplit(*_v); + _out.make_str.footer = _out.make.footer; } + return _out; } - - /+ ↓ what is this? by name, and for a directory by what it holds +/ - AbstractionSource abstractionSourceOf(string _path) { - if (_path.length == 0) { return AbstractionSource.unknown; } - if (_path.isValidPath && _path.exists && _path.isDir) { - return (_path.chainPath("pod.manifest").array.exists) - ? AbstractionSource.pod : AbstractionSource.unknown; + /+ ↓ where a document loaded from an artefact says it is. + an artefact names its document (% Source:) and its language (@source + language) but not where the pod holding its images sits, so that is + taken from where the artefact itself was found: + . + pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/ + pod/<uid>.ocda.db -> pod/<doc>/ + . + which is where --pod2 puts them both. The images beside a .ssp are + then found where html.d already looks; a .ocda.db carries its own and + does not need the directory to exist. + +/ + struct ST_ArtefactPaths { + string pod_dir; + string src_file_with_path; + } + ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) { + ST_ArtefactPaths _out; + string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en"; + string _dir = _artefact.dirName; + if (_artefact.endsWith(".ssp")) { + /+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/ + _out.pod_dir = (_dir.baseName == "abstraction") + ? _dir.dirName.dirName : _dir; + } else { + /+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/ + string _uid = _artefact.baseName; + foreach (_sfx; [".ocda.db", ".db"]) { + if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; } + } + if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; } + _out.pod_dir = (_dir.chainPath(_uid).array).to!string; } - if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } - if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } - if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } - if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } - if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } - return AbstractionSource.unknown; + _out.src_file_with_path = (_out.pod_dir + .chainPath("media").chainPath("text").chainPath(_lang) + .chainPath(_ssp.source).array).to!string; + return _out; } - - struct LoadedAbstraction { - AbstractionSource source; - string path; - bool loaded; // is .doc filled - string note; // why not, when it is not - SSPdocument doc; + /+ ↓ a database's images, written where the writers look for them. + returns the pod directory to use in place of the one beside the + artefact, or "" when the database carries no images and the pod + beside it should stand. Each blob is checked against the sha256 the + database recorded with it: the two were written together, so a + mismatch means the file is damaged, and saying so is the point of + having recorded the digest at all. + +/ + @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + import std.file : mkdirRecurse, write, tempDir; + import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; + mixin spineAbstractionDbRead _dbr; + auto _files = _dbr.dbReadFiles(_artefact, "image"); + if (_files.length == 0) { return ""; } + string _root = (tempDir.chainPath("spine-ocda-" + ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string; + string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string; + try { + _img_dir.mkdirRecurse; + } catch (Exception ex) { + stderr.writeln("WARNING: could not make an image directory for ", _artefact, + ": ", ex.msg); + return ""; + } + foreach (_f; _files) { + string _got = _f.data.sha256Of.toHexString.to!string; + if (_f.sha256.length > 0 && _got != _f.sha256) { + stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName, + " does not match the digest recorded with it (", _f.sha256, " expected, ", + _got, " found); it is written out as it stands"); + } + try { + (_img_dir.chainPath(_f.name).array).to!string.write(_f.data); + } catch (Exception ex) { + stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg); + } + } + if (_opt_action.vox_gt_1) { + writeln(" ", _files.length, " image(s) from the database, in ", _img_dir); + } + return _root; } - - /+ ↓ load what can be loaded from the path alone. - - .ssp and .ocda.db come back filled. the three source forms come back - with loaded = false and a note saying where they are handled, because - reading them needs the manifest, environment and configuration that - spine.d builds, not a file path. + /+ ↓ the images a .ssp describes, against the ones beside it. + a .ssp carries no image bytes, so what it can do is say whether the + files in the pod it sits in are the ones the abstraction was built + from. Nothing checked that before: a changed or wrong image in the + directory paired silently. +/ - LoadedAbstraction abstractionLoad(string _path) { - LoadedAbstraction _out; - _out.path = _path; - _out.source = abstractionSourceOf(_path); - if (!_path.exists) { - _out.note = "no such file or directory"; - return _out; + @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + import std.file : exists, read; + string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string; + int _checked, _missing, _wrong; + foreach (_section; _ssp.section_order) { + foreach (obj; _ssp.abstraction[_section]) { + foreach (_img; obj.metainfo.sha256.images) { + if (_img.fileName.length == 0) { continue; } + string _pth = (_img_dir.chainPath(_img.fileName).array).to!string; + if (!_pth.exists) { ++_missing; continue; } + ++_checked; + string _want = _img.fileHash_sha256.toHexString.to!string; + string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string; + if (_want != _got && _want != "00000000000000000000000000000000" + ~ "00000000000000000000000000000000") { + ++_wrong; + stderr.writeln("WARNING: image ", _img.fileName, " beside ", + _ssp.source, " is not the one the abstraction was built from"); + } + } + } } - final switch (_out.source) { - case AbstractionSource.ssp: - _out.doc = sspReadFile(_path); - _out.loaded = (_out.doc.section_order.length > 0); - if (!_out.loaded) { _out.note = "no object sections found"; } - break; - case AbstractionSource.ocda_db: - _out.doc = dbReadFile(_path); - _out.loaded = (_out.doc.section_order.length > 0); - if (!_out.loaded) { _out.note = "no object sections found"; } - break; - case AbstractionSource.markup: - case AbstractionSource.pod: - case AbstractionSource.pod_zip: - _out.note = "a source form: read by the parser," - ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the" - ~ " manifest, environment and configuration spine assembles"; - break; - case AbstractionSource.unknown: - _out.note = "not a document source spine knows"; - break; + if (_missing > 0) { + stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are", + " not in ", _img_dir); + } + if (_opt_action.vox_gt_1 && _checked > 0) { + writeln(" ", _checked, " image(s) checked against the abstraction's digests", + (_wrong == 0) ? ", all matching" : ""); + } + } + /+ ↓ the doc the output writers take, from an artefact. + the reading is done here rather than by the caller, and that is not + only tidiness: main() is @system, so a template mixed in there infers + @system for everything it declares, and the output writers are @safe + and will not take objects of such a type. Doing it inside this module, + which is @safe:, is what makes the loaded objects the same kind of + thing the parser's are. Called as spineDocFromArtefact!()(...), the + way spineAbstraction!() is, rather than mixed in. + +/ + auto spineDocFromArtefact(P,O,E)( + string _artefact, + P program_info, + O _opt_action, + E _env, + ConfComposite _site_conf, + ) { + auto _loaded = abstractionLoad(_artefact); + auto _ssp = _loaded.doc; + auto _pths = artefactPaths(_artefact, _ssp); + /+ ↓ the images. + a .ocda.db carries its own, which is what lets it travel on its + own, so they are taken from it and written where the output writers + look for images: a directory of this run's making, not the pod + beside the artefact, which may be somebody's published tree. A .ssp + only describes its images, so those are the ones in the pod it sits + in, and are checked against the digests it recorded. + +/ + string _images_tmp; + if (_loaded.loaded) { + if (_artefact.endsWith(".ssp")) { + _imagesVerify(_pths.pod_dir, _ssp, _opt_action); + } else { + string _extracted = _imagesExtract(_artefact, _opt_action); + if (_extracted.length > 0) { + /+ ↓ the source path has to move with it: image_dir_path is + reached from the document's own file, not from the pod +/ + _images_tmp = _extracted; + _pths.pod_dir = _extracted; + _pths.src_file_with_path = (_extracted + .chainPath("media").chainPath("text") + .chainPath(("language" in _ssp.source_info) + ? _ssp.source_info["language"] : "en") + .chainPath(_ssp.source).array).to!string; + } + } + } + auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); + auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); + auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); + /+ ↓ the digests: the markup digest is what @source carries, and it is + the one that says which source this abstraction was made from. The + header and text digests are of parts an artefact does not keep + separately, and are left unset rather than guessed at + +/ + ST_ArtefactDigest _dig; + if (auto _v = "digest" in _ssp.source_info) { + _dig.markup_doc = _hexToDigest(*_v); + } + auto _abstraction = _ssp.abstraction; + auto _matters = docMattersMake( + program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init, + ); + /+ ↓ the same shape spineAbstraction() returns, built the same way: a + struct of accessors over the locals, with whether the artefact + could be read at all, which the caller has to know +/ + auto theDOC() { + struct ST_DOC { + const auto abstraction() { return _abstraction; } + auto matters() { return _matters; } + bool loaded() { return _loaded.loaded; } + string images_tmp() { return _images_tmp; } + string note() { return _loaded.note; } + string source_name() { return abstractionSourceName(_loaded.source); } + } + return ST_DOC(); + } + return theDOC(); + } + /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from + an artefact, @source having recorded it; the header and text digests + are of parts no artefact keeps separately and are left unset rather + than guessed at. Shaped to match ST_doc_digest, which is declared + inside spineRawMarkupContent and reachable only by parsing a file. + +/ + struct ST_ArtefactDigest { + ubyte[32] markup_doc; + ubyte[32] header; + ubyte[32] text; + } + private ubyte[32] _hexToDigest(string _hex) { + ubyte[32] _out; + if (_hex.length < 64) { return _out; } + foreach (i; 0..32) { + try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {} } return _out; } +} +#+END_SRC - /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a - given artefact really does hold what it should +/ - string[] abstractionLoadSummary(LoadedAbstraction _l) { - string[] _out; - _out ~= "source: " ~ abstractionSourceName(_l.source); - _out ~= "path: " ~ _l.path; - if (!_l.loaded) { - _out ~= "loaded: no"; - if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; } - return _out; +* E. Get OCD + +get abstracted objects for downstream processing + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_has.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module sisudoc.ocda.abstraction.doc_has; +@safe: +/+ ↓ ST_DocHas from a loaded abstraction + + the parser builds ST_DocHas as it goes, with the markup in front of it. + A document loaded from a .ssp or a .ocda.db has to arrive at the same + value from what the artefact carries, and this is where that is done. + + Every one of these is an *index over properties the artefact already + holds*, not a re-derivation of something it does not: + + imagelist the .image records' filenames + segnames_lv4 .anchor of each body heading at level 4 + segnames_lv_0_to_4 .segment_epub of each heading at level 4 or less + tag_associations .anchor, .anchor_tag and .segment_* per object + section_keys_sequenced which sections are non-empty, plus the run's + own output flags + + That distinction is the line the .ssp skill draws: assembling an index + is fine and cannot drift, because a wrong index would not survive the + round trip that produced its inputs; recomputing dom_status or ancestors + would be re-derivation and is exactly what ocda exists to remove. If a + value is not in the file and cannot be indexed out of what is, it must + be added to the artefact instead of reconstructed here. + + The counts come from the @doc_has block rather than being recounted, for + the same reason. ++/ +template spineDocHasFromAbstraction() { + import std.algorithm : sort, uniq; + import std.array; + import std.conv : to; + import std.json : JSONValue; + import std.regex; + import sisudoc.ocda.meta.metadoc_object_setter; + import sisudoc.ocda.meta.rgx; + mixin ObjectSetter; + mixin spineRgxIn; + /+ ↓ the sections, in the order a document holds them +/ + enum string[] doc_sections = [ + "head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail", + ]; + /+ ↓ the tail sections that become segments of their own, in the order + after_doc_determine_segnames appends them +/ + enum string[] tail_sections = [ + "endnotes", "glossary", "bibliography", "bookindex", "blurb", + ]; + private uint _count(string[string] _doc_has, string _key) { + if (auto _v = _key in _doc_has) { + try { return (*_v).to!uint; } catch (Exception ex) { return 0; } } - _out ~= "loaded: yes"; - if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; } - if ("title.main" in _l.doc.meta) { - _out ~= "title: " ~ _l.doc.meta["title.main"]; + return 0; + } + private bool _has_section(A)(A _abst, string _section) { + if (auto _s = _section in _abst) { return (*_s).length > 1; } + return false; + } + /+ ↓ which document sections each output format walks, and in what order. + + the same rule the parser applies: the three that are always there, + then each tail section that the document actually has, then "tail" + for the formats that close with one. It depends on the run as well as + on the document, since only html and epub take the tail. + +/ + string[][string] sectionKeysSequenced(A,O)(A _abst, O _opt_action) { + string[][string] _keys = [ + "scroll": ["head", "toc", "body",], + "seg": ["head", "toc", "body",], + "sql": ["head", "body",], + "latex": ["head", "toc", "body",], + "text": ["head", "toc", "body",], + ]; + if (_has_section(_abst, "endnotes")) { + _keys["scroll"] ~= "endnotes"; + _keys["seg"] ~= "endnotes"; + _keys["latex"] ~= "endnotes"; + _keys["text"] ~= "endnotes"; } - if ("creator.author" in _l.doc.meta) { - _out ~= "author: " ~ _l.doc.meta["creator.author"]; + if (_has_section(_abst, "glossary")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "glossary"; } } - _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, " - ~ _l.doc.make.length.to!string ~ " make, " - ~ _l.doc.doc_has.length.to!string ~ " doc_has"; - int _objs, _citable, _headings; - string[] _sections; - foreach (section; _l.doc.section_order) { - int _n; - foreach (obj; _l.doc.abstraction[section]) { - ++_objs; ++_n; - if (obj.metainfo.ocn > 0) { ++_citable; } - if (obj.metainfo.is_a == "heading") { ++_headings; } + if (_has_section(_abst, "bibliography")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bibliography"; } + } + if (_has_section(_abst, "bookindex")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bookindex"; } + } + if (_has_section(_abst, "blurb")) { + foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "blurb"; } + } + if (_opt_action.html_scroll || _opt_action.html_seg || _opt_action.epub) { + _keys["scroll"] ~= "tail"; + _keys["seg"] ~= "tail"; + } + return _keys; + } + /+ ↓ the whole of it +/ + ST_DocHas docHasFromAbstraction(A,O)( + A _abst, + string[string] _doc_has_block, + O _opt_action, + ) { + static auto rgx = RgxI(); + ST_DocHas _out; + _out.inline_links = _count(_doc_has_block, "inline_links"); + _out.inline_notes_reg = _count(_doc_has_block, "inline_notes_reg"); + _out.inline_notes_star = _count(_doc_has_block, "inline_notes_star"); + _out.codeblocks = _count(_doc_has_block, "codeblocks"); + _out.tables = _count(_doc_has_block, "tables"); + _out.blocks = _count(_doc_has_block, "blocks"); + _out.groups = _count(_doc_has_block, "groups"); + _out.poems = _count(_doc_has_block, "poems"); + _out.quotes = _count(_doc_has_block, "quotes"); + string[] _images; + string[string][string] _tag_assoc; + /+ ↓ segnames["html"] is seeded with the toc, which is a segment of its + own before any level 4 heading opens one +/ + _out.segnames_lv4 ~= "toc"; + foreach (_section; doc_sections) { + if (_section !in _abst) { continue; } + /+ ↓ head is always walked; every other section only when it holds + more than the placeholder object, which is the guard the parser + puts on each of its section loops + +/ + if (_section != "head" && !_has_section(_abst, _section)) { continue; } + foreach (obj; _abst[_section]) { + /+ ↓ the images this object names, as the abstraction recorded them +/ + foreach (_img; obj.metainfo.sha256.images) { _images ~= _img.fileName; } + /+ ↓ every object: its html anchor is in a segment, and its epub + segment anchor stands for itself +/ + if (obj.tags.anchor_tag_html.length > 0) { + _tag_assoc[obj.tags.anchor_tag_html]["seg_lv4"] = obj.tags.in_segment_html; + } + if (obj.tags.segment_anchor_tag_epub.length > 0) { + _tag_assoc[obj.tags.segment_anchor_tag_epub]["seg_lv1to4"] + = obj.tags.segment_anchor_tag_epub; + } + /+ ↓ a heading's own anchor tags, and the anchors declared inline in + an object's text, both point at the segment the object is in +/ + /+ ↓ a heading above level 4 opens no html segment of its own, so a + cross reference to it has to land on the level 4 segment that + follows it. .segment is exactly that (the segment a heading + opens or falls into), where .segment_html_is is the segment an + object sits in; epub segments go down to level 1, so there + .segment_epub stands for the heading itself + +/ + if (obj.tags.heading_lev_anchor_tag.length > 0 + && obj.tags.in_segment_html.length > 0 + && obj.metainfo.heading_lev_markup >= 4 + ) { + _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv4"] + = obj.tags.in_segment_html; + if (obj.tags.segment_anchor_tag_epub.length > 0) { + _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv1to4"] + = obj.tags.segment_anchor_tag_epub; + } + } + foreach (m; obj.text.matchAll(rgx.inline_link_anchor)) { + string _a = m["anchor"].to!string; + if (_a.length == 0 || _a in _tag_assoc) { continue; } + _tag_assoc[_a]["seg_lv4"] = obj.tags.html_segment_anchor_tag_is; + _tag_assoc[_a]["seg_lv1to4"] = obj.tags.epub_segment_anchor_tag_is; + } + /+ ↓ the segment name lists, in document order +/ + if (obj.metainfo.is_a == "heading") { + if (obj.metainfo.heading_lev_markup <= 4) { + _out.segnames_lv_0_to_4 ~= obj.tags.segment_anchor_tag_epub; + } + if (_section == "body" && obj.metainfo.heading_lev_markup == 4) { + _out.segnames_lv4 ~= obj.tags.anchor_tag_html; + } + /+ ↓ a heading above level 4 opens no html segment of its own, so + a cross reference to it lands on the level 4 segment that + follows. That is .segment_lv4_is, resolved in ocda where the + finished sections are to hand and carried by the artefact, so + there is nothing to work out here + +/ + if (obj.tags.segment_lv4_is.length > 0) { + /+ ↓ seg_lv4 only, and not through heading_lev_anchor_tag: that + names the level 4 heading itself, whose own entry says it + opens that segment rather than merely sits in it. epub + segments go down to level 1, so seg_lv1to4 is untouched + +/ + foreach (_k; [obj.metainfo.identifier, obj.tags.anchor_tag_html]) { + if (_k.length > 0) { _tag_assoc[_k]["seg_lv4"] = obj.tags.segment_lv4_is; } + } + } + } } - _sections ~= section ~ " " ~ _n.to!string; } - _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string - ~ " citable, " ~ _headings.to!string ~ " headings)"; - _out ~= "sections: " ~ _sections.join(", "); + foreach (_section; tail_sections) { + if (_has_section(_abst, _section)) { _out.segnames_lv4 ~= _section; } + } + /+ ↓ the body's objects, in a pass of their own after every anchor above + has been recorded, because the parser does it in that order and the + order is what decides two of these entries. + . + each body object's identifier maps to the segment it is in. seg_lv4 + is set only where it is not already, an anchor of the same name + having said it *names* a segment rather than sits in one; seg_lv1to4 + is set always. A heading's own anchor can be a bare number taken + from its text ("2. Shorter Terms" gives the anchor "2"), which then + collides with the ocn of an unrelated object, and this is the order + that resolves the collision the way the parser resolves it: in + free_culture, ocn 2 keeps seg_lv4 "them" from the anchor and takes + seg_lv1to4 "_part_1" from itself. + +/ + if (_has_section(_abst, "body")) { + foreach (obj; _abst["body"]) { + if (obj.metainfo.identifier.length == 0) { continue; } + if (!((obj.metainfo.identifier in _tag_assoc) + && ("seg_lv4" in _tag_assoc[obj.metainfo.identifier])) + ) { + _tag_assoc[obj.metainfo.identifier]["seg_lv4"] + = obj.tags.html_segment_anchor_tag_is; + } + _tag_assoc[obj.metainfo.identifier]["seg_lv1to4"] + = obj.tags.epub_segment_anchor_tag_is; + } + } + _out.imagelist = _images.sort.uniq.array; + _out.tag_associations = _tag_assoc; + _out.section_keys_sequenced = sectionKeysSequenced(_abst, _opt_action); return _out; } } diff --git a/org/in_markup_source_files.org b/org/in_markup_source_files.org new file mode 100644 index 0000000..1b8cee0 --- /dev/null +++ b/org/in_markup_source_files.org @@ -0,0 +1,801 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) markup source raw +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:sourcefile:read: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] + +* A. get _config file_, read in (.readText) [#A] +** _module template_ :module:config_files: + +#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + read configuration files<BR> + - read config files<BR> + meta_config_files.d ++/ +module sisudoc.ocda.io_in.read_config_files; +@safe: +import std.file; +import std.path; +import sisudoc.ocda.meta; +import sisudoc.ocda.io_in.paths_source; +import sisudoc.ocda.meta.rgx_files; +import sisudoc.ocda.meta.rgx; +<<meta_config_file_hub_read_site_config>> +<<meta_config_file_hub_read_document_config>> +<<meta_config_file_hub_read_site_yaml>> +#+END_SRC + +*** read config files (config local site & dr document make) (yaml) +**** site configuration + +SEE NOTES on configuration hierarchy in spine.org + +#+NAME: meta_config_file_hub_read_site_config +#+BEGIN_SRC d +template readConfigSite() { + @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) { + mixin spineRgxIn; + static auto rgx = RgxI(); + string conf_filename = "NONE"; + string config_file_str; + string default_config_file_str = format(q"┃ +flag: + act0: "--html" + act1: "--html --epub" +output: + path: "%s" +default: + language: "en" + papersize: "a4" + text_wrap: "80" + digest: "sha256" +webserv: + http: "%s" + host: "%s" + data_http: "%s" + data_host: "%s" + data_root_url: "%s" + data_root_path: "%s" + data_root_part: "" + images_root_part: "image" + cgi_search_form_title: "%s" + cgi_http: "%s" + cgi_host: "%s" + cgi_bin_url: "%s" + cgi_bin_subpath: "%s" + cgi_bin_path: "%s" + cgi_search_script: "%s" + cgi_port: "" + cgi_user: "" + cgi_action: "%s" + db_sqlite_path: "%s" + db_sqlite_filename: "%s" + db_pg_table: "" + db_pg_user: "" +┃", + _cfg.processing_path_doc_root, // doc root + _cfg.http_request_type, // http + _cfg.http_host, // host / domain + _cfg.http_request_type, // data "http" or "https" + _cfg.http_host, // data domain "localhost" + _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org" + _cfg.processing_path_doc_root, // data root path + _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search" + _cfg.http_request_type, // cgi http + _cfg.http_host, // cgi host + _cfg.cgi_url_root, // cgi bin url + _cfg.cgi_bin_subpath, // cgi bin path + _cfg.cgi_bin_root, // cgi bin path + _cfg.cgi_filename, // cgi filename + _cfg.cgi_url_action, // cgi action + _cfg.db_sqlite_path, // sqlite db path + _cfg.db_sqlite_filename, // sqlite db filename +); + foreach(conf_fn; [_conf_file_details.config_filename_site]) { + foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) { + char[] conf_file; + conf_filename = conf_fn; + if (exists(pth)) { + auto f_attrib = pth.getLinkAttributes; + if ( + _conf_file_details.possible_config_path_locations.config_local_site.length == 1 + && f_attrib.attrIsFile + ) { + conf_file = pth.to!(char[]); + conf_filename = pth.baseName; + } else if (f_attrib.attrIsDir) { + conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; + conf_filename = conf_fn; + } + try { + if (exists(conf_file)) { + if (conf_file.getLinkAttributes.attrIsFile) { + if (_opt_action.vox_gt_2 || _opt_action.debug_do) { + writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)"); + } + config_file_str = conf_file.readText; + break; + } + } + } catch (ErrnoException ex) { + } catch (FileException ex) { + } + } + } + if (config_file_str.length > 0) { break; } + } + if (config_file_str.length > 0) { + import dyaml; + Node yaml_root; + try { + yaml_root = Loader.fromString(config_file_str).load(); + } catch (Throwable) { + import std.stdio; + writeln("ERROR failed to read config file content, not parsed as yaml, program default used"); + conf_filename = "VIRTUAL"; + config_file_str = default_config_file_str; + } + } + if (config_file_str.length == 0) { /+ use dummy default config file +/ + // writeln("WARNING config file NOT found, default provided"); + conf_filename = "VIRTUAL"; + config_file_str = default_config_file_str; + } + struct _ConfContent { + string filename() { + return conf_filename; + } + string filetype() { + string _ft = ""; + if (content.match(rgx.yaml_config)) { + _ft = "yaml"; + } + return _ft; + } + string content() { + return config_file_str; + } + } + return _ConfContent(); + } +} +#+END_SRC + +**** document make/config + +#+NAME: meta_config_file_hub_read_document_config +#+HEADER: :noweb yes +#+BEGIN_SRC d +static template readConfigDoc() { + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.meta.rgx_files; + import sisudoc.ocda.meta.rgx; + @system final auto readConfigDoc(M,E)(M _manifested, E _env) { + mixin spineRgxIn; + static auto rgx = RgxI(); + mixin spineRgxFiles; + static auto rgx_files = RgxFiles(); + string config_file_str; + string conf_filename = "NONE"; + auto _conf_file_details = configFilePaths!()(_manifested, _env); + string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; + foreach(conf_fn; [_conf_file_details.config_filename_document]) { + foreach(pth; possible_config_path_locations) { + char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; + conf_filename = conf_fn; + if (config_file_str.length > 0) { + break; + } + try { + if (exists(conf_file)) { + if (conf_file.getLinkAttributes.attrIsFile) { + config_file_str = conf_file.readText; + break; + } + } + } catch (ErrnoException ex) { + } catch (FileException ex) { + } + } + if (config_file_str.length > 0) { break; } + } + struct _ConfContent { + string filename() { + return conf_filename; + } + string content() { + return config_file_str; + } + string filetype() { + string _ft = ""; + if (content.match(rgx.yaml_config)) { + _ft = "yaml"; + } + return _ft; + } + } + return _ConfContent(); + } +} +#+END_SRC + +*** YAML config (config local site & dr document make) :file:config:hub: + +#+NAME: meta_config_file_hub_read_site_yaml +#+HEADER: :noweb yes +#+BEGIN_SRC d +static template configReadSiteYAML() { + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.meta.rgx_files; + import sisudoc.ocda.meta.rgx; + final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) { + string _configuration = configReadInSiteYAML!()(_manifested, _env); + auto _conf_file_details = configFilePaths!()(_manifested, _env); + string _conf_yaml_fn = _conf_file_details.config_filename_site; + YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); + return _yaml_conf; + } +} +static template configReadDocYAML() { + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) { + string _configuration = configReadInDocYAML!()(_manifested, _env); + auto _conf_file_details = configFilePaths!()(_manifested, _env); + string _conf_yaml_fn = _conf_file_details.config_filename_document; + YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); + return _yaml_conf; + } +} +#+END_SRC + +* B. get _markup source_, read file (.readText) [#A]:module:source_files: +** _module template_ + +#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + module source_read_source_files;<BR> + - open markup files<BR> + - if master file scan for addional files to import/insert ++/ +module sisudoc.ocda.io_in.read_source_files; +@safe: +template spineRawMarkupContent() { + import std.digest.sha; + import std.file; + import std.path; + import sisudoc.ocda.meta; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.meta.rgx_files; + import sisudoc.ocda.meta.rgx; + mixin spineRgxIn; + static auto rgx = RgxI(); + mixin spineRgxFiles; + static auto rgx_files = RgxFiles(); + string[] _images=[]; + /+ ↓ the images a markup source shows. + matched on the image markup itself ({ name.png }image, and the url and + ~^ forms), ensure a filename-shaped word in prose is not mistaken for + one, and over every match (so a line showing two images reports two). + +/ + string[] _extract_images(S)(S content_block) { + string[] images_; + string _content_block = content_block.to!string; + foreach (m; _content_block.matchAll(rgx.smid_image)) { + images_ ~= m["image"].to!string; + } + return images_; + } + auto rawsrc = RawMarkupContent(); + struct ST_contents_inserts_images { + char[][] contents; + string[] insert_files; + string[] images; + } + struct ST_header_content_inserts_images { + char[] header; + char[][] src_txt; + string[] insert_files; + string[] images; + } + struct ST_doc_digest { + ubyte[32] markup_doc; + ubyte[32] header; + ubyte[32] text; + } + struct ST_doc_parts { + char[] header_raw; + char[][] sourcefile_body_content; + string[] insert_file_list; + string[] images_list; + ST_doc_digest doc_digest; + } + ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) { + ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct + = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src); + return _0_header_1_body_content_2_insert_filelist_struct; + } + struct RawMarkupContent { + final sourceContent(in string fn_src) { + auto raw = MarkupRawUnit(); + string source_txt_str + = raw.markupSourceReadIn(fn_src); + return source_txt_str; + } + final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)( + O _opt_action, + in string source_txt_str, + in string fn_src="" + ) { + auto raw = MarkupRawUnit(); + string[] insert_file_list_get; + string[] images_list_get; + ST_header_content_inserts_images st + = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str); + char[] header_raw = st.header; + char[][] sourcefile_body_content = st.src_txt; + if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise + auto ins = Inserts(); + ST_contents_inserts_images _cii + = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); + sourcefile_body_content = _cii.contents; + insert_file_list_get = _cii.insert_files.dup; + images_list_get = _cii.images.dup; + } else if (_opt_action.source_or_pod) { + auto ins = Inserts(); + ST_contents_inserts_images _cii + = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); + images_list_get = _cii.images.dup; + } // image_list, if path could take sha256 digests already here? + string header_type = ""; + ST_doc_digest dig; + { + dig.markup_doc = source_txt_str.sha256Of; + dig.header = st.header.sha256Of; + dig.text = sourcefile_body_content.sha256Of; + } + ST_doc_parts ret; + { + ret.header_raw = st.header; + ret.sourcefile_body_content = sourcefile_body_content; + ret.insert_file_list = insert_file_list_get; + ret.images_list = images_list_get; + ret.doc_digest = dig; + } + return ret; + } + } + struct MarkupRawUnit { + import std.digest.sha; + import std.file; + <<meta_markup_source_raw_read_file_source_string>> + <<meta_markup_source_raw_doc_header_and_content_split>> + <<meta_markup_source_raw_source_line_array>> + <<meta_markup_source_raw_read_in_file>> + <<meta_markup_source_raw_struct_of_header_and_body>> + <<meta_markup_source_raw_get_insert_source_line_array>> + } + struct Inserts { + struct ST_contents_and_images { + char[][] insert_contents; + string[] images; + } + ST_contents_and_images scan_subdoc_source(O)( + O _opt_action, + char[][] markup_sourcefile_insert_content, + string fn_src + ) { + <<meta_inserts_scan>> + foreach (line; markup_sourcefile_insert_content) { + <<meta_inserts_scan_loop>> + } // end src subdoc (inserts) loop + <<meta_inserts_scan_post>> + } + ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)( + O _opt_action, + char[][] sourcefile_body_content, + string fn_src + ) { + import std.algorithm; + <<meta_master_doc_scan_for_insert_filenames>> + foreach (line; sourcefile_body_content) { + <<meta_master_doc_scan_for_insert_filenames_loop>> + } // end src doc loop + <<meta_master_doc_scan_for_insert_filenames_post>> + } + } +} +#+END_SRC + +** get markup source, read file :source:markup: +*** read file, source string [#A] :string: + +#+NAME: meta_markup_source_raw_read_file_source_string +#+BEGIN_SRC d +final private string readInMarkupSource(in char[] fn_src) { + enforce( + exists(fn_src) != 0, + "file not found: «" ~ + fn_src ~ "»" + ); + string source_txt_str; + try { + if (exists(fn_src)) { + if (fn_src.getLinkAttributes.attrIsFile) { + source_txt_str = fn_src.readText; + } else { + } + } + } catch (ErrnoException ex) { + } catch (UTFException ex) { + // Handle validation errors + } catch (FileException ex) { + // Handle errors + } + std.utf.validate(source_txt_str); + return source_txt_str; +} +#+END_SRC + +*** document header & content, array.length == 2 [#A] :array: + +here you split document header and body, an array.length == 2 +split is on first match of level A~ (which is required) + +#+NAME: meta_markup_source_raw_doc_header_and_content_split +#+BEGIN_SRC d +@trusted final private char[][] header0Content1(in string src_text) { // cast(char[]) + /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ + char[][] header_and_content; + auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a); + header_and_content ~= m.pre; + header_and_content ~= m.hit ~ m.post; + assert(header_and_content.length == 2, + "document markup is broken, header body split == " + ~ header_and_content.length.to!string + ~ "; (header / body array split should == 2 (split is on level A~))" + ); + return header_and_content; +} +#+END_SRC + +*** source line array :array: + +#+NAME: meta_markup_source_raw_source_line_array +#+BEGIN_SRC d +@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[]) + char[][] source_line_arr + = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding); + return source_line_arr; +} +#+END_SRC + +*** source content raw line array :array: +- used for regular .sst files; master .ssm files and; .ssi inserts +- regex is passed for relevant enforce match +**** read in file + +#+NAME: meta_markup_source_raw_read_in_file +#+BEGIN_SRC d +string markupSourceReadIn(in string fn_src) { + static auto rgx_files = RgxFiles(); + enforce( + fn_src.match(rgx_files.src_pth_sst_or_ssm), + "not a dr markup filename: «" ~ + fn_src ~ "»" + ); + string source_txt_str = readInMarkupSource(fn_src); + return source_txt_str; +} +#+END_SRC + +**** tuple (a) header, (b) body content, (c) file insert list & (d) image list? + +- header +- body content +- file insert list +- [image list?] + +#+NAME: meta_markup_source_raw_struct_of_header_and_body +#+BEGIN_SRC d +ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) { + string[] file_insert_list = []; + string[] images_list = []; + char[][] hc = header0Content1(source_txt_str); + char[] header = hc[0]; + char[] source_txt = hc[1]; + char[][] source_line_arr = markupSourceLineArray(source_txt); + ST_header_content_inserts_images ret; + { + ret.header = header; + ret.src_txt = source_line_arr; + ret.insert_files = file_insert_list; + ret.images = images_list; + } + return ret; +} +#+END_SRC + +**** get insert source line array + +#+NAME: meta_markup_source_raw_get_insert_source_line_array +#+BEGIN_SRC d +final char[][] getInsertMarkupSourceContentRawLineArray( + in char[] fn_src_insert, + Regex!(char) rgx_file +) { + enforce( + fn_src_insert.match(rgx_file), + "not a dr markup filename: «" ~ + fn_src_insert ~ "»" + ); + string source_txt_str = readInMarkupSource(fn_src_insert); + char[][] source_line_arr = markupSourceLineArray(source_txt_str); + return source_line_arr; +} +#+END_SRC + +** get markup source, master file & inserts :masterfile:inserts: +*** scan inserts (sub-document) source :scan_insert_src: +**** scan subdoc source + +#+NAME: meta_inserts_scan +#+BEGIN_SRC d +char[][] contents_insert; +int code_block_status = 0; +enum codeBlock { off, curly, tic, quotemarks } +auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); +auto markup_src_file_path = fn_pth_full.captures[1]; +#+END_SRC + +**** loop insert (sub-document) + +#+NAME: meta_inserts_scan_loop +#+BEGIN_SRC d +if (code_block_status == codeBlock.curly) { + if (line.matchFirst(rgx.block_curly_code_close)) { + code_block_status = codeBlock.off; + } + contents_insert ~= line; +} else if (line.matchFirst(rgx.block_curly_code_open)) { + code_block_status = codeBlock.curly; + contents_insert ~= line; +} else if (code_block_status == codeBlock.quotemarks) { + if (line.matchFirst(rgx.block_quotemarks_close)) { + code_block_status = codeBlock.off; + } + contents_insert ~= line; +} else if (code_block_status == codeBlock.tic) { + if (line.matchFirst(rgx.block_tic_close)) { + code_block_status = codeBlock.off; + } + contents_insert ~= line; +} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { + code_block_status = codeBlock.quotemarks; + contents_insert ~= line; +} else if (line.matchFirst(rgx.block_tic_code_open)) { + code_block_status = codeBlock.tic; + contents_insert ~= line; +} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { + auto insert_fn = m.captures[2]; + auto insert_sub_pth = m.captures[1]; + auto fn_src_insert + = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; + auto raw = MarkupRawUnit(); + auto markup_sourcesubfile_insert_content + = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); + debug(insert_file) { + writeln(line); + writeln(fn_src_insert); + writeln( + " length contents insert array: ", + markup_sourcesubfile_insert_content.length + ); + } + if (_opt_action.source_or_pod) { + _images ~= _extract_images(markup_sourcesubfile_insert_content); + } + auto ins = Inserts(); + /+ + - 1. load file + - 2. read lines + - 3. scan lines + - a. if filename insert, and insert filename + - repeat 1 + - b. else + - add line to new array; + - build image list, search for any image files to add to image list + +/ +} else { + contents_insert ~= line; // images to extract for image list? + if (_opt_action.source_or_pod) { + string[] _image_linelist = _extract_images(line); + if (_image_linelist.length > 0) { + _images ~= _image_linelist; + } + } +} +#+END_SRC + +**** post loop + +#+NAME: meta_inserts_scan_post +#+BEGIN_SRC d +ST_contents_and_images ret; +{ + ret.insert_contents = contents_insert; + ret.images = _images; +} +return ret; +#+END_SRC + +*** scan document source :scan_src: +**** scan doc source + +#+NAME: meta_master_doc_scan_for_insert_filenames +#+BEGIN_SRC d +char[][] contents; +int code_block_status = 0; +enum codeBlock { off, curly, tic, quotemarks } +auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); +auto markup_src_file_path = fn_pth_full.captures[1]; +char[][] contents_insert; +string[] _images =[]; +string[] insert_file_list =[]; +#+END_SRC + +**** include inserts: _loop master_ scan for inserts (insert documents) + +#+NAME: meta_master_doc_scan_for_insert_filenames_loop +#+BEGIN_SRC d +if (code_block_status == codeBlock.curly) { + if (line.matchFirst(rgx.block_curly_code_close)) { + code_block_status = codeBlock.off; + } + contents ~= line; +} else if (line.matchFirst(rgx.block_curly_code_open)) { + code_block_status = codeBlock.curly; + contents ~= line; +} else if (code_block_status == codeBlock.quotemarks) { + if (line.matchFirst(rgx.block_quotemarks_close)) { + code_block_status = codeBlock.off; + } + contents ~= line; +} else if (code_block_status == codeBlock.tic) { + if (line.matchFirst(rgx.block_tic_close)) { + code_block_status = codeBlock.off; + } + contents ~= line; +} else if (line.matchFirst(rgx.block_quotemarks_code_open)) { + code_block_status = codeBlock.quotemarks; + contents ~= line; +} else if (line.matchFirst(rgx.block_tic_code_open)) { + code_block_status = codeBlock.tic; + contents ~= line; +} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { + auto insert_fn = m.captures[2]; + auto insert_sub_pth = m.captures[1]; + auto fn_src_insert + = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; + insert_file_list ~= fn_src_insert.to!string; + auto raw = MarkupRawUnit(); + /+ TODO +/ + auto markup_sourcefile_insert_content + = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); + debug(insert_file) { + writeln(line); + writeln(fn_src_insert); + writeln( + " length contents insert array: ", + markup_sourcefile_insert_content.length + ); + } + auto ins = Inserts(); + ST_contents_and_images contents_insert_st = ins.scan_subdoc_source( + _opt_action, + markup_sourcefile_insert_content, + fn_src_insert.to!string + ); + contents ~= contents_insert_st.insert_contents; + if (_opt_action.source_or_pod) { + string[] _image_linelist = _extract_images(contents_insert_st.images); + if (_image_linelist.length > 0) { + _images ~= _image_linelist; + } + } + /+ + - 1. load file + - 2. read lines + - 3. scan lines + - a. if filename insert, and insert filename + - repeat 1 + - b. else + - add line to new array; + - build image list, search for any image files to add to image list + +/ +} else { + contents ~= line; + if (_opt_action.source_or_pod) { + string[] _image_linelist = _extract_images(line); + if (_image_linelist.length > 0) { + _images ~= _image_linelist; + } + } +} +#+END_SRC + +**** post loop + +#+NAME: meta_master_doc_scan_for_insert_filenames_post +#+BEGIN_SRC d +string[] images = []; +foreach(i; uniq(_images.sort())) { + images ~= i; +} +debug(insert_file) { + writeln(__LINE__); + writeln(contents.length); +} +ST_contents_inserts_images ret; +{ + ret.contents = contents; + ret.insert_files = insert_file_list; + ret.images = images; +} +return ret; +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/in_source_artefact.org b/org/in_source_artefact.org deleted file mode 100644 index 07c865f..0000000 --- a/org/in_source_artefact.org +++ /dev/null @@ -1,392 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) markup source raw -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:sourcefile:read: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* A. get doc from artefact -** _module template_ :module:config_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module sisudoc.ocda.abstraction.doc_from_artefact; -@safe: -/+ ↓ a document, from an artefact rather than from markup - spineAbstraction() parses markup and hands the output writers a doc: - an abstraction and a doc_matters. This does the same from a .ssp or a - .ocda.db, filling the same docMattersMake() arguments, so that what - comes out is the value the writers already take and they never learn - which of the two made it. - . - Three of the seven arguments come from the run and are the same either - way: program_info, opt_action, and the conf half of conf_make_meta, - which is site and run scoped (urls, papersize, the search database) and - has no business in a document's own artefact. The other four come from - the artefact: - . - conf_make_meta meta and make from the @meta and @make blocks, over - the site config the run assembled - doc_has docHasFromAbstraction, over the objects - doc_digest the markup digest from @source - manifest synthesised from the artefact's path and the - document's own name and language - . - What is not here, deliberately: insert_file_list, which is read only by - source_pod.d for --source and --pod2. No artefact carries the markup, - so a pod cannot be rebuilt from one, and the empty list is the truthful - answer rather than a gap. -+/ -template spineDocFromArtefact() { - import std.algorithm : endsWith; - import std.array; - import std.conv : to; - import std.path; - import std.process : thisProcessID; - import std.regex; - import std.stdio; - import std.string; - import sisudoc.ocda.meta.conf_make_meta_structs; - import sisudoc.ocda.meta.doc_matters; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.abstraction.doc_has; - import sisudoc.ocda.abstraction.load; - import sisudoc.ocda.meta.topic_register; - mixin spineDocMatters; - mixin spineDocHasFromAbstraction; - mixin spineAbstractionLoad; - mixin spineTopicRegister; - /+ ↓ a .ssp header key from a field name: the first underscore becomes the - dot that separates the group, so title_main is title.main and - rights_copyright_text is rights.copyright_text. The writer's keys are - formed the same way, which is what lets this be read off the struct - rather than kept as a second list to fall out of step with it. - +/ - private string _sspKeyOf(string _member) { - foreach (i, c; _member) { - if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; } - } - return _member; - } - /+ ↓ the document's own header, over the site config the run assembled. - every string field of MetaComposite is looked for under its own key, - so a field added there and emitted by the writer arrives here without - anything being added. The arr fields are split from the string they - come from, as agreed: that is parsing a value, not deriving a - structure. - +/ - ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) { - ConfComposite _out = _site_conf; // conf: site and run scoped, kept - static foreach (_m; __traits(allMembers, MetaComposite)) { - static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{ - enum string _key = _sspKeyOf(_m); - if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; } - }} - } - /+ ↓ title_sub is a copy of title_subtitle, made where the header is - read, and is what epub3 puts in dc:title id="subtitle". It is not - emitted, being the same string twice; it is remade here the way the - yaml reader makes it +/ - _out.meta.title_sub = _out.meta.title_subtitle; - /+ ↓ creator.author is the author list joined with ", ", so splitting on - it gives the list back +/ - if (_out.meta.creator_author.length > 0) { - foreach (_a; _out.meta.creator_author.split(", ")) { - if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; } - } - } - /+ ↓ the topic register arrays, by the one rule the yaml reader uses: - the split is not a plain one and deriving it twice would be two - answers to the same question +/ - { - auto _tr = topicRegisterArrays(_out.meta.classify_topic_register); - _out.meta.classify_topic_register_arr = _tr.arr; - _out.meta.classify_topic_register_expanded_arr = _tr.expanded; - } - /+ ↓ make: the keys are the field names, with no group to separate +/ - if (auto _v = "doc_type" in _ssp.make) { - _out.make.doc_type = *_v; _out.make_str.doc_type = *_v; - } - if (auto _v = "auto_num_top_at_level" in _ssp.make) { - _out.make.auto_num_top_at_level = *_v; - } - if (auto _v = "auto_num_top_lv" in _ssp.make) { - try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {} - } - if (auto _v = "auto_num_depth" in _ssp.make) { - try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {} - } - if (auto _v = "breaks" in _ssp.make) { - _out.make.breaks = *_v; _out.make_str.breaks = *_v; - } - if (auto _v = "home_button_text" in _ssp.make) { - _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v; - } - /+ ↓ footer is a list, joined with the separator the format escapes +/ - if (auto _v = "footer" in _ssp.make) { - _out.make.footer = sspUnescapeSplit(*_v); - _out.make_str.footer = _out.make.footer; - } - return _out; - } - /+ ↓ where a document loaded from an artefact says it is. - an artefact names its document (% Source:) and its language (@source - language) but not where the pod holding its images sits, so that is - taken from where the artefact itself was found: - . - pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/ - pod/<uid>.ocda.db -> pod/<doc>/ - . - which is where --pod2 puts them both. The images beside a .ssp are - then found where html.d already looks; a .ocda.db carries its own and - does not need the directory to exist. - +/ - struct ST_ArtefactPaths { - string pod_dir; - string src_file_with_path; - } - ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) { - ST_ArtefactPaths _out; - string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en"; - string _dir = _artefact.dirName; - if (_artefact.endsWith(".ssp")) { - /+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/ - _out.pod_dir = (_dir.baseName == "abstraction") - ? _dir.dirName.dirName : _dir; - } else { - /+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/ - string _uid = _artefact.baseName; - foreach (_sfx; [".ocda.db", ".db"]) { - if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; } - } - if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; } - _out.pod_dir = (_dir.chainPath(_uid).array).to!string; - } - _out.src_file_with_path = (_out.pod_dir - .chainPath("media").chainPath("text").chainPath(_lang) - .chainPath(_ssp.source).array).to!string; - return _out; - } - /+ ↓ a database's images, written where the writers look for them. - returns the pod directory to use in place of the one beside the - artefact, or "" when the database carries no images and the pod - beside it should stand. Each blob is checked against the sha256 the - database recorded with it: the two were written together, so a - mismatch means the file is damaged, and saying so is the point of - having recorded the digest at all. - +/ - @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - import std.file : mkdirRecurse, write, tempDir; - import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; - mixin spineAbstractionDbRead _dbr; - auto _files = _dbr.dbReadFiles(_artefact, "image"); - if (_files.length == 0) { return ""; } - string _root = (tempDir.chainPath("spine-ocda-" - ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string; - string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string; - try { - _img_dir.mkdirRecurse; - } catch (Exception ex) { - stderr.writeln("WARNING: could not make an image directory for ", _artefact, - ": ", ex.msg); - return ""; - } - foreach (_f; _files) { - string _got = _f.data.sha256Of.toHexString.to!string; - if (_f.sha256.length > 0 && _got != _f.sha256) { - stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName, - " does not match the digest recorded with it (", _f.sha256, " expected, ", - _got, " found); it is written out as it stands"); - } - try { - (_img_dir.chainPath(_f.name).array).to!string.write(_f.data); - } catch (Exception ex) { - stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg); - } - } - if (_opt_action.vox_gt_1) { - writeln(" ", _files.length, " image(s) from the database, in ", _img_dir); - } - return _root; - } - /+ ↓ the images a .ssp describes, against the ones beside it. - a .ssp carries no image bytes, so what it can do is say whether the - files in the pod it sits in are the ones the abstraction was built - from. Nothing checked that before: a changed or wrong image in the - directory paired silently. - +/ - @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - import std.file : exists, read; - string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string; - int _checked, _missing, _wrong; - foreach (_section; _ssp.section_order) { - foreach (obj; _ssp.abstraction[_section]) { - foreach (_img; obj.metainfo.sha256.images) { - if (_img.fileName.length == 0) { continue; } - string _pth = (_img_dir.chainPath(_img.fileName).array).to!string; - if (!_pth.exists) { ++_missing; continue; } - ++_checked; - string _want = _img.fileHash_sha256.toHexString.to!string; - string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string; - if (_want != _got && _want != "00000000000000000000000000000000" - ~ "00000000000000000000000000000000") { - ++_wrong; - stderr.writeln("WARNING: image ", _img.fileName, " beside ", - _ssp.source, " is not the one the abstraction was built from"); - } - } - } - } - if (_missing > 0) { - stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are", - " not in ", _img_dir); - } - if (_opt_action.vox_gt_1 && _checked > 0) { - writeln(" ", _checked, " image(s) checked against the abstraction's digests", - (_wrong == 0) ? ", all matching" : ""); - } - } - /+ ↓ the doc the output writers take, from an artefact. - the reading is done here rather than by the caller, and that is not - only tidiness: main() is @system, so a template mixed in there infers - @system for everything it declares, and the output writers are @safe - and will not take objects of such a type. Doing it inside this module, - which is @safe:, is what makes the loaded objects the same kind of - thing the parser's are. Called as spineDocFromArtefact!()(...), the - way spineAbstraction!() is, rather than mixed in. - +/ - auto spineDocFromArtefact(P,O,E)( - string _artefact, - P program_info, - O _opt_action, - E _env, - ConfComposite _site_conf, - ) { - auto _loaded = abstractionLoad(_artefact); - auto _ssp = _loaded.doc; - auto _pths = artefactPaths(_artefact, _ssp); - /+ ↓ the images. - a .ocda.db carries its own, which is what lets it travel on its - own, so they are taken from it and written where the output writers - look for images: a directory of this run's making, not the pod - beside the artefact, which may be somebody's published tree. A .ssp - only describes its images, so those are the ones in the pod it sits - in, and are checked against the digests it recorded. - +/ - string _images_tmp; - if (_loaded.loaded) { - if (_artefact.endsWith(".ssp")) { - _imagesVerify(_pths.pod_dir, _ssp, _opt_action); - } else { - string _extracted = _imagesExtract(_artefact, _opt_action); - if (_extracted.length > 0) { - /+ ↓ the source path has to move with it: image_dir_path is - reached from the document's own file, not from the pod +/ - _images_tmp = _extracted; - _pths.pod_dir = _extracted; - _pths.src_file_with_path = (_extracted - .chainPath("media").chainPath("text") - .chainPath(("language" in _ssp.source_info) - ? _ssp.source_info["language"] : "en") - .chainPath(_ssp.source).array).to!string; - } - } - } - auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); - auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); - auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); - /+ ↓ the digests: the markup digest is what @source carries, and it is - the one that says which source this abstraction was made from. The - header and text digests are of parts an artefact does not keep - separately, and are left unset rather than guessed at - +/ - ST_ArtefactDigest _dig; - if (auto _v = "digest" in _ssp.source_info) { - _dig.markup_doc = _hexToDigest(*_v); - } - auto _abstraction = _ssp.abstraction; - auto _matters = docMattersMake( - program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init, - ); - /+ ↓ the same shape spineAbstraction() returns, built the same way: a - struct of accessors over the locals, with whether the artefact - could be read at all, which the caller has to know +/ - auto theDOC() { - struct ST_DOC { - const auto abstraction() { return _abstraction; } - auto matters() { return _matters; } - bool loaded() { return _loaded.loaded; } - string images_tmp() { return _images_tmp; } - string note() { return _loaded.note; } - string source_name() { return abstractionSourceName(_loaded.source); } - } - return ST_DOC(); - } - return theDOC(); - } - /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from - an artefact, @source having recorded it; the header and text digests - are of parts no artefact keeps separately and are left unset rather - than guessed at. Shaped to match ST_doc_digest, which is declared - inside spineRawMarkupContent and reachable only by parsing a file. - +/ - struct ST_ArtefactDigest { - ubyte[32] markup_doc; - ubyte[32] header; - ubyte[32] text; - } - private ubyte[32] _hexToDigest(string _hex) { - ubyte[32] _out; - if (_hex.length < 64) { return _out; } - foreach (i; 0..32) { - try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {} - } - return _out; - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/ocda_get.org b/org/ocda_get.org deleted file mode 100644 index 40effc5..0000000 --- a/org/ocda_get.org +++ /dev/null @@ -1,284 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) object-centric document abstraction -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:abstraction: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* Get OCD - -get abstracted objects for downstream processing - -** _module template_ - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_has.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module sisudoc.ocda.abstraction.doc_has; -@safe: -/+ ↓ ST_DocHas from a loaded abstraction - - the parser builds ST_DocHas as it goes, with the markup in front of it. - A document loaded from a .ssp or a .ocda.db has to arrive at the same - value from what the artefact carries, and this is where that is done. - - Every one of these is an *index over properties the artefact already - holds*, not a re-derivation of something it does not: - - imagelist the .image records' filenames - segnames_lv4 .anchor of each body heading at level 4 - segnames_lv_0_to_4 .segment_epub of each heading at level 4 or less - tag_associations .anchor, .anchor_tag and .segment_* per object - section_keys_sequenced which sections are non-empty, plus the run's - own output flags - - That distinction is the line the .ssp skill draws: assembling an index - is fine and cannot drift, because a wrong index would not survive the - round trip that produced its inputs; recomputing dom_status or ancestors - would be re-derivation and is exactly what ocda exists to remove. If a - value is not in the file and cannot be indexed out of what is, it must - be added to the artefact instead of reconstructed here. - - The counts come from the @doc_has block rather than being recounted, for - the same reason. -+/ -template spineDocHasFromAbstraction() { - import std.algorithm : sort, uniq; - import std.array; - import std.conv : to; - import std.json : JSONValue; - import std.regex; - import sisudoc.ocda.meta.metadoc_object_setter; - import sisudoc.ocda.meta.rgx; - mixin ObjectSetter; - mixin spineRgxIn; - /+ ↓ the sections, in the order a document holds them +/ - enum string[] doc_sections = [ - "head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", "tail", - ]; - /+ ↓ the tail sections that become segments of their own, in the order - after_doc_determine_segnames appends them +/ - enum string[] tail_sections = [ - "endnotes", "glossary", "bibliography", "bookindex", "blurb", - ]; - private uint _count(string[string] _doc_has, string _key) { - if (auto _v = _key in _doc_has) { - try { return (*_v).to!uint; } catch (Exception ex) { return 0; } - } - return 0; - } - private bool _has_section(A)(A _abst, string _section) { - if (auto _s = _section in _abst) { return (*_s).length > 1; } - return false; - } - /+ ↓ which document sections each output format walks, and in what order. - - the same rule the parser applies: the three that are always there, - then each tail section that the document actually has, then "tail" - for the formats that close with one. It depends on the run as well as - on the document, since only html and epub take the tail. - +/ - string[][string] sectionKeysSequenced(A,O)(A _abst, O _opt_action) { - string[][string] _keys = [ - "scroll": ["head", "toc", "body",], - "seg": ["head", "toc", "body",], - "sql": ["head", "body",], - "latex": ["head", "toc", "body",], - "text": ["head", "toc", "body",], - ]; - if (_has_section(_abst, "endnotes")) { - _keys["scroll"] ~= "endnotes"; - _keys["seg"] ~= "endnotes"; - _keys["latex"] ~= "endnotes"; - _keys["text"] ~= "endnotes"; - } - if (_has_section(_abst, "glossary")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "glossary"; } - } - if (_has_section(_abst, "bibliography")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bibliography"; } - } - if (_has_section(_abst, "bookindex")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "bookindex"; } - } - if (_has_section(_abst, "blurb")) { - foreach (_k; ["scroll", "seg", "sql", "latex", "text"]) { _keys[_k] ~= "blurb"; } - } - if (_opt_action.html_scroll || _opt_action.html_seg || _opt_action.epub) { - _keys["scroll"] ~= "tail"; - _keys["seg"] ~= "tail"; - } - return _keys; - } - /+ ↓ the whole of it +/ - ST_DocHas docHasFromAbstraction(A,O)( - A _abst, - string[string] _doc_has_block, - O _opt_action, - ) { - static auto rgx = RgxI(); - ST_DocHas _out; - _out.inline_links = _count(_doc_has_block, "inline_links"); - _out.inline_notes_reg = _count(_doc_has_block, "inline_notes_reg"); - _out.inline_notes_star = _count(_doc_has_block, "inline_notes_star"); - _out.codeblocks = _count(_doc_has_block, "codeblocks"); - _out.tables = _count(_doc_has_block, "tables"); - _out.blocks = _count(_doc_has_block, "blocks"); - _out.groups = _count(_doc_has_block, "groups"); - _out.poems = _count(_doc_has_block, "poems"); - _out.quotes = _count(_doc_has_block, "quotes"); - string[] _images; - string[string][string] _tag_assoc; - /+ ↓ segnames["html"] is seeded with the toc, which is a segment of its - own before any level 4 heading opens one +/ - _out.segnames_lv4 ~= "toc"; - foreach (_section; doc_sections) { - if (_section !in _abst) { continue; } - /+ ↓ head is always walked; every other section only when it holds - more than the placeholder object, which is the guard the parser - puts on each of its section loops - +/ - if (_section != "head" && !_has_section(_abst, _section)) { continue; } - foreach (obj; _abst[_section]) { - /+ ↓ the images this object names, as the abstraction recorded them +/ - foreach (_img; obj.metainfo.sha256.images) { _images ~= _img.fileName; } - /+ ↓ every object: its html anchor is in a segment, and its epub - segment anchor stands for itself +/ - if (obj.tags.anchor_tag_html.length > 0) { - _tag_assoc[obj.tags.anchor_tag_html]["seg_lv4"] = obj.tags.in_segment_html; - } - if (obj.tags.segment_anchor_tag_epub.length > 0) { - _tag_assoc[obj.tags.segment_anchor_tag_epub]["seg_lv1to4"] - = obj.tags.segment_anchor_tag_epub; - } - /+ ↓ a heading's own anchor tags, and the anchors declared inline in - an object's text, both point at the segment the object is in +/ - /+ ↓ a heading above level 4 opens no html segment of its own, so a - cross reference to it has to land on the level 4 segment that - follows it. .segment is exactly that (the segment a heading - opens or falls into), where .segment_html_is is the segment an - object sits in; epub segments go down to level 1, so there - .segment_epub stands for the heading itself - +/ - if (obj.tags.heading_lev_anchor_tag.length > 0 - && obj.tags.in_segment_html.length > 0 - && obj.metainfo.heading_lev_markup >= 4 - ) { - _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv4"] - = obj.tags.in_segment_html; - if (obj.tags.segment_anchor_tag_epub.length > 0) { - _tag_assoc[obj.tags.heading_lev_anchor_tag]["seg_lv1to4"] - = obj.tags.segment_anchor_tag_epub; - } - } - foreach (m; obj.text.matchAll(rgx.inline_link_anchor)) { - string _a = m["anchor"].to!string; - if (_a.length == 0 || _a in _tag_assoc) { continue; } - _tag_assoc[_a]["seg_lv4"] = obj.tags.html_segment_anchor_tag_is; - _tag_assoc[_a]["seg_lv1to4"] = obj.tags.epub_segment_anchor_tag_is; - } - /+ ↓ the segment name lists, in document order +/ - if (obj.metainfo.is_a == "heading") { - if (obj.metainfo.heading_lev_markup <= 4) { - _out.segnames_lv_0_to_4 ~= obj.tags.segment_anchor_tag_epub; - } - if (_section == "body" && obj.metainfo.heading_lev_markup == 4) { - _out.segnames_lv4 ~= obj.tags.anchor_tag_html; - } - /+ ↓ a heading above level 4 opens no html segment of its own, so - a cross reference to it lands on the level 4 segment that - follows. That is .segment_lv4_is, resolved in ocda where the - finished sections are to hand and carried by the artefact, so - there is nothing to work out here - +/ - if (obj.tags.segment_lv4_is.length > 0) { - /+ ↓ seg_lv4 only, and not through heading_lev_anchor_tag: that - names the level 4 heading itself, whose own entry says it - opens that segment rather than merely sits in it. epub - segments go down to level 1, so seg_lv1to4 is untouched - +/ - foreach (_k; [obj.metainfo.identifier, obj.tags.anchor_tag_html]) { - if (_k.length > 0) { _tag_assoc[_k]["seg_lv4"] = obj.tags.segment_lv4_is; } - } - } - } - } - } - foreach (_section; tail_sections) { - if (_has_section(_abst, _section)) { _out.segnames_lv4 ~= _section; } - } - /+ ↓ the body's objects, in a pass of their own after every anchor above - has been recorded, because the parser does it in that order and the - order is what decides two of these entries. - . - each body object's identifier maps to the segment it is in. seg_lv4 - is set only where it is not already, an anchor of the same name - having said it *names* a segment rather than sits in one; seg_lv1to4 - is set always. A heading's own anchor can be a bare number taken - from its text ("2. Shorter Terms" gives the anchor "2"), which then - collides with the ocn of an unrelated object, and this is the order - that resolves the collision the way the parser resolves it: in - free_culture, ocn 2 keeps seg_lv4 "them" from the anchor and takes - seg_lv1to4 "_part_1" from itself. - +/ - if (_has_section(_abst, "body")) { - foreach (obj; _abst["body"]) { - if (obj.metainfo.identifier.length == 0) { continue; } - if (!((obj.metainfo.identifier in _tag_assoc) - && ("seg_lv4" in _tag_assoc[obj.metainfo.identifier])) - ) { - _tag_assoc[obj.metainfo.identifier]["seg_lv4"] - = obj.tags.html_segment_anchor_tag_is; - } - _tag_assoc[obj.metainfo.identifier]["seg_lv1to4"] - = obj.tags.epub_segment_anchor_tag_is; - } - } - _out.imagelist = _images.sort.uniq.array; - _out.tag_associations = _tag_assoc; - _out.section_keys_sequenced = sectionKeysSequenced(_abst, _opt_action); - return _out; - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/out_src_abstraction_ocda_peg_ssp.org b/org/out_ocda_peg_ssp.org index 511c589..511c589 100644 --- a/org/out_src_abstraction_ocda_peg_ssp.org +++ b/org/out_ocda_peg_ssp.org diff --git a/org/out_src_abstraction_sqlite_db.org b/org/out_ocda_sqlite_db.org index 70c118a..70c118a 100644 --- a/org/out_src_abstraction_sqlite_db.org +++ b/org/out_ocda_sqlite_db.org |
