From 6491006a7e64574605a4fe85c2e37ce36aec4eb3 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 27 Apr 2020 11:30:39 -0400 Subject: rename & house utils; work on markup conversion - new dir for markup tools misc/util/d/tools/markup_conversion - start looking at tools for converting - endnotes from binary to inline - sisu.rb to sisu spine in d - look at conversion of document headers to yaml from - bespoke sisu headers (original .rb) - sdlang - toml --- misc/util/d/tools/endnotes_inline_from_binary.d | 127 -------- misc/util/d/tools/markup_conversion/README | 1 + .../endnotes_inline_from_binary.d | 127 ++++++++ .../d/tools/markup_conversion/markup_changes.d | 136 ++++++++ .../markup_changes_header_and_content.d | 244 ++++++++++++++ ...arkup_conversion_from_sisu_ruby_to_sisu_spine.d | 354 +++++++++++++++++++++ 6 files changed, 862 insertions(+), 127 deletions(-) delete mode 100755 misc/util/d/tools/endnotes_inline_from_binary.d create mode 100644 misc/util/d/tools/markup_conversion/README create mode 100755 misc/util/d/tools/markup_conversion/endnotes_inline_from_binary.d create mode 100644 misc/util/d/tools/markup_conversion/markup_changes.d create mode 100755 misc/util/d/tools/markup_conversion/markup_changes_header_and_content.d create mode 100755 misc/util/d/tools/markup_conversion/markup_conversion_from_sisu_ruby_to_sisu_spine.d (limited to 'misc') diff --git a/misc/util/d/tools/endnotes_inline_from_binary.d b/misc/util/d/tools/endnotes_inline_from_binary.d deleted file mode 100755 index 4d9ceb0..0000000 --- a/misc/util/d/tools/endnotes_inline_from_binary.d +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env rdmd -/+ - - read in file .sst .ssi .ssm - - loop twice - - first - - check for and skip code blocks - - use unique code marker for endnote markers in text and give an endnote - number ★1, increment - - extract all endnotes in array - - second - - check that the footnote marker number count matches the number of notes - in the array - - if they match either: - - substitute each endnote marker with the array footnote[number-1] - - substitute each endnote marker with footnote - as inlined footnote markup (footnote number not needed) - - if they do not match exit - - check whether changes have been made - - if so write file with inline footnotes in sub-directory converted_output_/ - using the same name as the original file - - else, exit -+/ -import std.stdio; -import std.file; -import std.array : split; -import std.exception; -import core.stdc.errno; -import std.regex; -import std.format; -import std.conv; -void main(string[] args) { - foreach(arg; args[1..$]) { - if ( - !(arg.match(regex(r"--\w+"))) - && arg.match(regex(r"\w+?\.ss[itm]")) - ) { - writeln(arg); - string filename = arg; - try { - string[] contents, endnotes, endnote_refs; - string text = filename.readText; - string[] paragraphs = text.split("\n\n"); - int endnote_ref_count = 0; - int[string] type = [ - "curly_code" : 0, - "tic_code" : 0, - ]; - static comment = ctRegex!(`^%+ `); - static block_tic_code_open = ctRegex!("^`{3} code(?:[.](?P[a-z][0-9a-z#+_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?"); - static block_tic_close = ctRegex!("^(`{3})$","m"); - static block_curly_code_open = ctRegex!(`^(?:code(?:[.](?P[a-z][0-9a-z_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?[{][ ]*$)`); - static block_curly_code_close = ctRegex!(`^([}]code)`); - auto rgx_endnote_ref = ctRegex!(`([~]\^)(?P[)\]]? |$)`, "gm"); - auto rgx_endnote = ctRegex!(`^\^~\s+(.+|\n)`, "gm"); - foreach (paragraph; paragraphs) { /+ loop to gather binary endnotes +/ - if ( !( type["curly_code"] == 1 || type["tic_code"] == 1) - && paragraph.match(rgx_endnote) - ) { - endnotes ~= replaceAll!(m => m[1]) - (paragraph, rgx_endnote); - } else { - if ( type["curly_code"] == 1 - && paragraph.matchFirst(block_curly_code_close) - ) { - type["curly_code"] = 0; - } else if (type["tic_code"] == 1 - && paragraph.matchFirst(block_tic_close) - ) { - type["tic_code"] = 0; - } else if ( type["curly_code"] == 1 || type["tic_code"] == 1) { - // prevent search for endnotes - } else if (paragraph.matchFirst(block_curly_code_open)) { - type["curly_code"] = 1; - } else if (paragraph.matchFirst(block_tic_code_open)) { - type["tic_code"] = 1; - } else if (auto m = paragraph.matchAll(rgx_endnote_ref)) { - foreach (n; m) { - endnote_ref_count++; // endnote_refs ~= (n.captures[1]); - } - } - contents ~= paragraph; - } - } - if (endnotes.length == endnote_ref_count) { - import std.outbuffer; - writeln("endnote ref count: ", endnote_ref_count); - writeln("number of binary endnotes: ", endnotes.length); - int endnote_count = -1; - auto buffer = new OutBuffer(); - foreach (content; contents) { /+ loop to inline endnotes +/ - content = replaceAll!(m => "~{ " ~ endnotes[++endnote_count] ~ " }~" ~ m["tail"] ) - (content, rgx_endnote_ref); - buffer.write(content ~ "\n\n"); - } - if (buffer) { - try { - string dir_out = "converted_output_"; - string path_and_file_out = dir_out ~ "/" ~ filename; - dir_out.mkdirRecurse; - auto f = File(path_and_file_out, "w"); - f.write(buffer); - writeln("wrote: ", path_and_file_out); - } catch (FileException ex) { - writeln("did not write file"); - // Handle errors - } - } - } else { - writeln("ERROR binary endnote mismatch, check markup,\nmisatch in the number of endnotes & endnote references!"); - writeln(" number of endnotes: ", endnotes.length); - writeln(" number of endnote refs: ", endnote_ref_count); // endnote_refs.length, - } - // assert(endnotes.length == endnote_ref_count); - } catch (ErrnoException ex) { - switch(ex.errno) { - case EPERM: - case EACCES: // Permission denied - break; - case ENOENT: // File does not exist - break; - default: // Handle other errors - break; - } - } - } - } -} diff --git a/misc/util/d/tools/markup_conversion/README b/misc/util/d/tools/markup_conversion/README new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/misc/util/d/tools/markup_conversion/README @@ -0,0 +1 @@ + diff --git a/misc/util/d/tools/markup_conversion/endnotes_inline_from_binary.d b/misc/util/d/tools/markup_conversion/endnotes_inline_from_binary.d new file mode 100755 index 0000000..abd4e45 --- /dev/null +++ b/misc/util/d/tools/markup_conversion/endnotes_inline_from_binary.d @@ -0,0 +1,127 @@ +#!/usr/bin/env rdmd +/+ + - read in file .sst .ssi .ssm + - loop twice + - first + - check for and skip code blocks + - use unique code marker for endnote markers in text and give an endnote + number ★1, increment + - extract all endnotes in array + - second + - check that the footnote marker number count matches the number of notes + in the array + - if they match either: + - substitute each endnote marker with the array footnote[number-1] + - substitute each endnote marker with footnote + as inlined footnote markup (footnote number not needed) + - if they do not match exit + - check whether changes have been made + - if so write file with inline footnotes in sub-directory converted_output_/ + using the same name as the original file + - else, exit ++/ +import std.stdio; +import std.file; +import std.array : split; +import std.exception; +import core.stdc.errno; +import std.regex; +import std.format; +import std.conv; +void main(string[] args) { + static comment = ctRegex!(`^%+ `); + static block_tic_code_open = ctRegex!("^`{3} code(?:[.](?P[a-z][0-9a-z#+_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?"); + static block_tic_close = ctRegex!("^(`{3})$","m"); + static block_curly_code_open = ctRegex!(`^(?:code(?:[.](?P[a-z][0-9a-z_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?[{][ ]*$)`); + static block_curly_code_close = ctRegex!(`^([}]code)`); + auto rgx_endnote_ref = ctRegex!(`([~]\^)(?P[)\]]? |$)`, "gm"); + auto rgx_endnote = ctRegex!(`^\^~\s+(.+|\n)`, "gm"); + foreach(arg; args[1..$]) { + if ( + !(arg.match(regex(r"--\w+"))) + && arg.match(regex(r"\w+?\.ss[itm]")) + ) { + writeln(arg); + string filename = arg; + try { + string[] contents, endnotes, endnote_refs; + string text = filename.readText; + string[] paragraphs = text.split("\n\n"); + int endnote_ref_count = 0; + int[string] type = [ + "curly_code" : 0, + "tic_code" : 0, + ]; + foreach (paragraph; paragraphs) { /+ loop to gather binary endnotes +/ + if ( !( type["curly_code"] == 1 || type["tic_code"] == 1) + && paragraph.match(rgx_endnote) + ) { + endnotes ~= replaceAll!(m => m[1]) + (paragraph, rgx_endnote); + } else { + if ( type["curly_code"] == 1 + && paragraph.matchFirst(block_curly_code_close) + ) { + type["curly_code"] = 0; + } else if (type["tic_code"] == 1 + && paragraph.matchFirst(block_tic_close) + ) { + type["tic_code"] = 0; + } else if ( type["curly_code"] == 1 || type["tic_code"] == 1) { + // skip, prevent search for endnotes + } else if (paragraph.matchFirst(block_curly_code_open)) { + type["curly_code"] = 1; + } else if (paragraph.matchFirst(block_tic_code_open)) { + type["tic_code"] = 1; + } else if (auto m = paragraph.matchAll(rgx_endnote_ref)) { + foreach (n; m) { + endnote_ref_count++; // endnote_refs ~= (n.captures[1]); + } + } + contents ~= paragraph; + } + } + if (endnotes.length == endnote_ref_count) { + import std.outbuffer; + writeln("endnote ref count: ", endnote_ref_count); + writeln("number of binary endnotes: ", endnotes.length); + int endnote_count = -1; + auto buffer = new OutBuffer(); + foreach (content; contents) { /+ loop to inline endnotes +/ + content = replaceAll!(m => "~{ " ~ endnotes[++endnote_count] ~ " }~" ~ m["tail"] ) + (content, rgx_endnote_ref); + buffer.write(content ~ "\n\n"); + } + if (buffer) { + try { + string dir_out = "converted_output_"; + string path_and_file_out = dir_out ~ "/" ~ filename; + dir_out.mkdirRecurse; + auto f = File(path_and_file_out, "w"); + f.write(buffer); + writeln("wrote: ", path_and_file_out); + } catch (FileException ex) { + writeln("did not write file"); + // Handle errors + } + } + } else { + writeln("ERROR binary endnote mismatch, check markup,\nmisatch in the number of endnotes & endnote references!"); + writeln(" number of endnotes: ", endnotes.length); + writeln(" number of endnote refs: ", endnote_ref_count); // endnote_refs.length, + } + // assert(endnotes.length == endnote_ref_count); + } catch (ErrnoException ex) { + switch(ex.errno) { + case EPERM: + case EACCES: // Permission denied + break; + case ENOENT: // File does not exist + break; + default: // Handle other errors + break; + } + } + } + } +} diff --git a/misc/util/d/tools/markup_conversion/markup_changes.d b/misc/util/d/tools/markup_conversion/markup_changes.d new file mode 100644 index 0000000..4274f78 --- /dev/null +++ b/misc/util/d/tools/markup_conversion/markup_changes.d @@ -0,0 +1,136 @@ +#!/usr/bin/env rdmd +/+ + - read in file .sst .ssi .ssm + - loop twice + - first + - check for and skip code blocks + - use unique code marker for endnote markers in text and give an endnote + number ★1, increment + - extract all endnotes in array + - second + - check that the footnote marker number count matches the number of notes + in the array + - if they match either: + - substitute each endnote marker with the array footnote[number-1] + - substitute each endnote marker with footnote + as inlined footnote markup (footnote number not needed) + - if they do not match exit + - check whether changes have been made + - if so write file with inline footnotes in sub-directory converted_output_/ + using the same name as the original file + - else, exit ++/ +import std.stdio; +import std.file; +import std.array : split; +import std.exception; +import core.stdc.errno; +import std.regex; +import std.format; +import std.conv; +void main(string[] args) { + static comment = ctRegex!(`^%+ `); + static block_tic_code_open = ctRegex!("^`{3} code(?:[.](?P[a-z][0-9a-z#+_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?"); + static block_tic_close = ctRegex!("^(`{3})$","m"); + static block_curly_code_open = ctRegex!(`^(?:code(?:[.](?P[a-z][0-9a-z_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?[{][ ]*$)`); + static block_curly_code_close = ctRegex!(`^([}]code)`); + auto rgx_endnote_ref = ctRegex!(`([~]\^)(?P[)\]]? |$)`, "gm"); + auto rgx_endnote = ctRegex!(`^\^~\s+(.+|\n)`, "gm"); + foreach(arg; args[1..$]) { + if ( + !(arg.match(regex(r"--\w+"))) + && arg.match(regex(r"\w+?\.ss[itm]")) + ) { + writeln(arg); + string filename = arg; + try { + string[] contents, endnotes, endnote_refs; + string text = filename.readText; + string[] paragraphs = text.split("\n\n"); + int endnote_ref_count = 0; + int[string] type = [ + "curly_code" : 0, + "tic_code" : 0, + ]; + foreach (paragraph; paragraphs) { /+ loop to gather binary endnotes +/ + if ( !( type["curly_code"] == 1 || type["tic_code"] == 1) + && paragraph.match(rgx_endnote) + ) { + endnotes ~= replaceAll!(m => m[1]) + (paragraph, rgx_endnote); + } else { + if ( type["curly_code"] == 1 || type["tic_code"] == 1 + || paragraph.matchFirst(block_curly_code_open) + || paragraph.matchFirst(block_tic_code_open) + ) { /+ code blocks identified, no munging +/ + if ( type["curly_code"] == 1 + && paragraph.matchFirst(block_curly_code_close) + ) { + type["curly_code"] = 0; + } else if (type["tic_code"] == 1 + && paragraph.matchFirst(block_tic_close) + ) { + type["tic_code"] = 0; + } else if (paragraph.matchFirst(block_curly_code_open)) { + type["curly_code"] = 1; + } else if (paragraph.matchFirst(block_tic_code_open)) { + type["tic_code"] = 1; + } + contents ~= paragraph; + } else { /+ regular content, not a code block +/ + if (auto m = paragraph.matchAll(rgx_endnote_ref)) { + foreach (n; m) { + endnote_ref_count++; // endnote_refs ~= (n.captures[1]); + } + } + paragraph = replaceAll!(m => " \\\\ " ) + (paragraph, regex(r"\s*<(?:/\s*|:)?br>\s*")); // (paragraph, regex(r"(
)")); + contents ~= paragraph; + } + } + } + { + import std.outbuffer; + auto buffer = new OutBuffer(); + if (endnotes.length == endnote_ref_count) { + // writeln("endnote ref count: ", endnote_ref_count); + // writeln("number of binary endnotes: ", endnotes.length); + int endnote_count = -1; + foreach (content; contents) { /+ loop to inline endnotes +/ + content = replaceAll!(m => "~{ " ~ endnotes[++endnote_count] ~ " }~" ~ m["tail"] ) + (content, rgx_endnote_ref); // endnote_ref cannot occur in a code block or else fail + buffer.write(content ~ "\n\n"); + } + if (buffer) { + try { + string dir_out = "converted_output_"; + string path_and_file_out = dir_out ~ "/" ~ filename; + dir_out.mkdirRecurse; + auto f = File(path_and_file_out, "w"); + f.write(buffer); + writeln("wrote: ", path_and_file_out); + } catch (FileException ex) { + writeln("did not write file"); + // Handle errors + } + } + } else { + foreach (content; contents) { /+ loop to inline endnotes +/ + buffer.write(content ~ "\n\n"); + } + } + } + } catch (ErrnoException ex) { + switch(ex.errno) { + case EPERM: + case EACCES: // Permission denied + break; + case ENOENT: // File does not exist + break; + default: // Handle other errors + break; + } + } + } + } +} diff --git a/misc/util/d/tools/markup_conversion/markup_changes_header_and_content.d b/misc/util/d/tools/markup_conversion/markup_changes_header_and_content.d new file mode 100755 index 0000000..86792ff --- /dev/null +++ b/misc/util/d/tools/markup_conversion/markup_changes_header_and_content.d @@ -0,0 +1,244 @@ +#!/usr/bin/env rdmd +/+ + - read in file .sst .ssi .ssm + - loop twice + - first + - check for and skip code blocks + - use unique code marker for endnote markers in text and give an endnote + number ★1, increment + - extract all endnotes in array + - second + - check that the footnote marker number count matches the number of notes + in the array + - if they match either: + - substitute each endnote marker with the array footnote[number-1] + - substitute each endnote marker with footnote + as inlined footnote markup (footnote number not needed) + - if they do not match exit + - check whether changes have been made + - if so write file with inline footnotes in sub-directory converted_output_/ + using the same name as the original file + - else, exit ++/ +import std.stdio; +import std.file; +import std.array : split; +import std.exception; +// import std.range; +import core.stdc.errno; +import std.regex; +import std.format; +import std.conv; +void main(string[] args) { + static heading_a = ctRegex!(`^:?[A][~] `, "m"); + static comment = ctRegex!(`^%+ `); + static block_tic_code_open = ctRegex!("^`{3} code(?:[.](?P[a-z][0-9a-z#+_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?"); + static block_tic_close = ctRegex!("^(`{3})$","m"); + static block_curly_code_open = ctRegex!(`^(?:code(?:[.](?P[a-z][0-9a-z_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?[{][ ]*$)`); + static block_curly_code_close = ctRegex!(`^([}]code)`); + auto rgx_endnote_ref = ctRegex!(`([~]\^)(?P[)\]]? |$)`, "gm"); + auto rgx_endnote = ctRegex!(`^\^~\s+(.+|\n)`, "gm"); + char[][] header0Content1(in string src_text) { // cast(char[]) + /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ + char[][] header_and_content; + auto m = (cast(char[]) src_text).matchFirst(heading_a); + header_and_content ~= m.pre; + header_and_content ~= m.hit ~ m.post; + assert(header_and_content.length == 2, + "document markup is broken, header body split == " + ~ header_and_content.length.to!string + ~ "; (header / body array split should == 2 (split is on level A~))" + ); + return header_and_content; + } + foreach(arg; args[1..$]) { + if ( + !(arg.match(regex(r"--\w+"))) + && arg.match(regex(r"\w+?\.ss[itm]")) + ) { + writeln(arg); + string filename = arg; + try { + string[] munged_header, munged_contents, munged_endnotes, endnote_refs; + string text = filename.readText; + char[][] hc = header0Content1(text); + char[] src_header = hc[0]; + string[] headers = src_header.to!string.split("\n\n"); + char[] src_txt = hc[1]; + string[] paragraphs = src_txt.to!string.split("\n\n"); + int endnote_ref_count = 0; + int[string] type = [ + "curly_code" : 0, + "tic_code" : 0, + ]; + string _tmp_header; + foreach (h_; headers) { /+ loop to inline endnotes +/ + _tmp_header = ""; + if (h_.match(regex(r"^[@\[]?title[:\]]?"))) { // title + if (auto m = h_.match(regex(r"^@title:(?:\s+(?P.+)|$)"))) { // sisu bespoke markup + if (m.captures["c"].length == 0) { + _tmp_header ~= "title:"; + } else { + _tmp_header ~= "title:\n main: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } else if (auto m = h_.match(regex(r"^title\s*=\s*(?P.+)"))) { // toml? + if (m.captures["c"].length == 0) { + _tmp_header ~= "title:"; + } else { + _tmp_header ~= "title:\n main: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } else if (auto m = h_.match(regex(r"^\[title\]"))) { // toml markup + _tmp_header ~= "title:"; + } else if (auto m = h_.match(regex(r"^title(?:\s+(?P.+)|\s+\\$)"))) { // sdlang markup + if (m.captures["c"].length == 0) { + _tmp_header ~= "title:"; + } else { + _tmp_header ~= "title:\n main: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } + if (h_.match(regex(r"^\s*[:]?(?:main)[:= ]?", "m"))) { + if (auto m = h_.match(regex(r"^\s+(?P:main):(?:\s+(?P.+)|$)", "m"))) { // sisu bespoke markup + _tmp_header ~= " main: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s*(?Pmain)\s*=\s*(?P.+)", "m"))) { // toml? + _tmp_header ~= " main: " ~ m.captures["c"]; + } else if (auto m = h_.match(regex(r"^\s+(?Pmain)(?:\s*\s*(?P.+)|$)", "m"))) { // toml markup + _tmp_header ~= " main: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s+(?Pmain)(?:\s+(?P.+)|\s+\\$)", "m"))) { // sdlang markup + _tmp_header ~= " main: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } + if (h_.match(regex(r"^\s*[:]?(?:sub(title)?)[:= ]?", "m"))) { + if (auto m = h_.match(regex(r"^\s+:sub(?:title)?:(?:\s+(?P.+)|$)", "m"))) { // sisu bespoke markup + _tmp_header ~= " subtitle: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s*sub(?:title)?\s*=\s*(?P.+)$", "m"))) { // toml? + _tmp_header ~= " subtitle: " ~ m.captures["c"]; + } else if (auto m = h_.match(regex(r"^\s+(?:title)?(?:\s*\s*(?P.+)|$)", "m"))) { // toml markup + _tmp_header ~= " subtitle: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s+(?:title)?(?:\s+(?P.+)|\s+\\$)", "m"))) { // sdlang markup + _tmp_header ~= " subtitle: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } + } + if (h_.match(regex(r"^[@\[]?rights[:\]]?"))) { // rights + if (auto m = h_.match(regex(r"^@rights:[ ]+(?P.+)$"))) { // sisu bespoke markup + _tmp_header ~= "rights: \n copyright: \"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^@rights:"))) { // sisu bespoke markup + _tmp_header ~= "rights:"; + } else if (auto m = h_.match(regex(r"^\[rights\]", "m"))) { // toml markup + _tmp_header ~= "rights:"; + } else if (auto m = h_.match(regex(r"^rights:"))) { // sdlang markup + _tmp_header ~= "rights:"; + } + if (h_.match(regex(r"^\s*[:]?copyright[:= ]?", "m"))) { + if (auto m = h_.match(regex(r"^\s+:copyright:(?:\s+(?P.+)|$)", "m"))) { // sisu bespoke markup + _tmp_header ~= " copyright: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s*copyright\s*=\s*(?P.+)", "m"))) { // toml? + _tmp_header ~= " copyright: " ~ m.captures["c"]; + } else if (auto m = h_.match(regex(r"^\s+copyright(?:\s*\s*(?P.+)|$)", "m"))) { // toml markup + _tmp_header ~= " copyright: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s+copyright(?:\s+(?P.+)|\s+\\$)", "m"))) { // sdlang markup + _tmp_header ~= " copyright: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } + if (h_.match(regex(r"^\s*[:]?licen[cs]e[:= ]?", "m"))) { + if (auto m = h_.match(regex(r"^\s+:licen[cs]e:(?:\s+(?P.+)|$)", "m"))) { // sisu bespoke markup + _tmp_header ~= " license: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s*licen[cs]e\s*=\s*(?P.+)$", "m"))) { // toml? + _tmp_header ~= " license: " ~ m.captures["c"]; + } else if (auto m = h_.match(regex(r"^\s+licen[cs]e(?:\s*\s*(?P.+)|$)", "m"))) { // toml markup + _tmp_header ~= " license: " ~ "\"" ~ m.captures["c"] ~ "\""; + } else if (auto m = h_.match(regex(r"^\s+licen[cs]e(?:\s+(?P.+)|\s+\\$)", "m"))) { // sdlang markup + _tmp_header ~= " license: " ~ "\"" ~ m.captures["c"] ~ "\""; + } + } + } + if (_tmp_header.length > 0) { + munged_header ~= _tmp_header; + } else { + munged_header ~= h_; + } + } + writeln(munged_header); + foreach (paragraph; paragraphs) { /+ loop to gather binary endnotes +/ + if ( !( type["curly_code"] == 1 || type["tic_code"] == 1) + && paragraph.match(rgx_endnote) + ) { + munged_endnotes ~= replaceAll!(m => m[1]) + (paragraph, rgx_endnote); + } else { + if ( type["curly_code"] == 1 || type["tic_code"] == 1 + || paragraph.matchFirst(block_curly_code_open) + || paragraph.matchFirst(block_tic_code_open) + ) { /+ code blocks identified, no munging +/ + if ( type["curly_code"] == 1 + && paragraph.matchFirst(block_curly_code_close) + ) { + type["curly_code"] = 0; + } else if (type["tic_code"] == 1 + && paragraph.matchFirst(block_tic_close) + ) { + type["tic_code"] = 0; + } else if (paragraph.matchFirst(block_curly_code_open)) { + type["curly_code"] = 1; + } else if (paragraph.matchFirst(block_tic_code_open)) { + type["tic_code"] = 1; + } + munged_contents ~= paragraph; + } else { /+ regular content, not a code block +/ + if (auto m = paragraph.matchAll(rgx_endnote_ref)) { + foreach (n; m) { + endnote_ref_count++; // endnote_refs ~= (n.captures[1]); + } + } + paragraph = replaceAll!(m => " \\\\ " ) + (paragraph, regex(r"\s*<(?:/\s*|:)?br>\s*")); // (paragraph, regex(r"(
)")); + munged_contents ~= paragraph; + } + } + } + { + import std.outbuffer; + auto buffer = new OutBuffer(); + foreach (header; munged_header) { /+ loop to inline endnotes +/ + buffer.write(header ~ "\n\n"); + } + if (munged_endnotes.length == endnote_ref_count) { + int endnote_count = -1; + foreach (content; munged_contents) { /+ loop to inline endnotes +/ + content = replaceAll!(m => "~{ " ~ munged_endnotes[++endnote_count] ~ " }~" ~ m["tail"] ) + (content, rgx_endnote_ref); // endnote_ref cannot occur in a code block or else fail + buffer.write(content ~ "\n\n"); + } + if (buffer) { + try { + string dir_out = "converted_output_"; + string path_and_file_out = dir_out ~ "/" ~ filename; + dir_out.mkdirRecurse; + auto f = File(path_and_file_out, "w"); + f.write(buffer); + // writeln("wrote: ", path_and_file_out); + } catch (FileException ex) { + writeln("did not write file"); + // Handle errors + } + } + } else { + foreach (content; munged_contents) { /+ loop to inline endnotes +/ + buffer.write(content ~ "\n\n"); + } + } + } + } catch (ErrnoException ex) { + switch(ex.errno) { + case EPERM: + case EACCES: // Permission denied + break; + case ENOENT: // File does not exist + break; + default: // Handle other errors + break; + } + } + } + } +} diff --git a/misc/util/d/tools/markup_conversion/markup_conversion_from_sisu_ruby_to_sisu_spine.d b/misc/util/d/tools/markup_conversion/markup_conversion_from_sisu_ruby_to_sisu_spine.d new file mode 100755 index 0000000..94e8718 --- /dev/null +++ b/misc/util/d/tools/markup_conversion/markup_conversion_from_sisu_ruby_to_sisu_spine.d @@ -0,0 +1,354 @@ +#!/usr/bin/env rdmd +/+ + - read in file .sst .ssi .ssm + - loop twice + - first + - check for and skip code blocks + - use unique code marker for endnote markers in text and give an endnote + number ★1, increment + - extract all endnotes in array + - second + - check that the footnote marker number count matches the number of notes + in the array + - if they match either: + - substitute each endnote marker with the array footnote[number-1] + - substitute each endnote marker with footnote + as inlined footnote markup (footnote number not needed) + - if they do not match exit + - check whether changes have been made + - if so write file with inline footnotes in sub-directory converted_output_/ + using the same name as the original file + - else, exit ++/ +import std.stdio; +import std.file; +import std.array : split, join; +import std.exception; +// import std.range; +import core.stdc.errno; +import std.regex; +import std.format; +import std.conv; +void main(string[] args) { + static heading_a = ctRegex!(`^:?[A][~] `, "m"); + static comment = ctRegex!(`^%+ `); + static block_tic_code_open = ctRegex!("^`{3} code(?:[.](?P[a-z][0-9a-z#+_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?"); + static block_tic_close = ctRegex!("^(`{3})$","m"); + static block_curly_code_open = ctRegex!(`^(?:code(?:[.](?P[a-z][0-9a-z_]+))?(?:[(](?P[ a-zA-Z0-9;:,]*)[)])?[{][ ]*$)`); + static block_curly_code_close = ctRegex!(`^([}]code)`); + auto rgx_endnote_ref = ctRegex!(`([~]\^)(?P[)\]]? |$)`, "gm"); + auto rgx_endnote = ctRegex!(`^\^~\s+(.+|\n)`, "gm"); + char[][] header0Content1(in string src_text) { // cast(char[]) + /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ + char[][] header_and_content; + auto m = (cast(char[]) src_text).matchFirst(heading_a); + header_and_content ~= m.pre; + header_and_content ~= m.hit ~ m.post; + assert(header_and_content.length == 2, + "document markup is broken, header body split == " + ~ header_and_content.length.to!string + ~ "; (header / body array split should == 2 (split is on level A~))" + ); + return header_and_content; + } + string format_body_string(string s) { + string o; + o = s + .replaceAll(regex("^<(?:/[ ]*)?br>[ ]*"), " \\\\ ") + .replaceAll(regex("[ ]*<(?:/[ ]*)?br>$"), " \\\\") + .replaceAll(regex("[ ]*<(?:/[ ]*)?br>[ ]*"), " \\\\ "); + return o; + } + string format_header_string(string s) { + string o; + o = s + .replaceAll(regex("\""), "\\\"") + .replaceAll(regex("[ ]*<(?:/[ ]*)?br>$"), " \\\\") + .replaceAll(regex("[ ]*<(?:/[ ]*)?br>[ ]*"), " \\\\ "); + return o; + } + string format_main_header(string hm, string hs = "", string c = "") { + string o; + if (c.length == 0) { + o ~= hm ~ ":\n"; + } else { + o ~= hm ~ ":\n" + ~ " " ~ hs ~ ": " + ~ "\"" ~ format_header_string(c) ~ "\"\n"; + } + return o; + } + string format_sub_header(string hs, string c) { + string o; + o ~= " " ~ hs ~ ": " + ~ "\"" ~ format_header_string(c) ~ "\"\n"; + return o; + } + foreach(arg; args[1..$]) { + if ( + !(arg.match(regex(r"--\w+"))) + && arg.match(regex(r"\w+?\.ss[itm]")) + ) { + writeln(arg); + string filename = arg; + try { + string[] munged_header, munged_contents, munged_endnotes, endnote_refs; + string text = filename.readText; + char[][] hc = header0Content1(text); + char[] src_header = hc[0]; + string[] headers = src_header.to!string.split("\n\n"); + char[] src_txt = hc[1]; + string[] paragraphs = src_txt.to!string.split("\n\n"); + int endnote_ref_count = 0; + int[string] type = [ + "curly_code" : 0, + "tic_code" : 0, + ]; + string _tmp_header; + headers[0] = headers[0].replaceFirst(regex(r"^%\s+SiSU.+", "i"), "# SiSU 8.0 spine (auto-conversion)"); + foreach (h_; headers) { + _tmp_header = ""; + if (auto m = h_.match(regex(r"^%\s*", "m"))) { + h_ = h_.replaceAll(regex(r"^%\s*", "m"), "# ") ~ "\n"; + } + if (h_.match(regex(r"^@title:|@subtitle"))) { + if (auto m = h_.match(regex(r"^@(?Ptitle):(?:[ ]+(?P.+)|\n)"))) { + _tmp_header ~= format_main_header(m.captures["h"], "main", m.captures["c"]); + } + if (auto m = h_.match(regex(r"^@(?Psubtitle):(?:[ ]+(?P.+)|$)"))) { + if (m.captures["c"].length == 0) { + } else { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + } + if (auto m = h_.match(regex(r"^\s+:(?Pmain):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:sub(?:title)?:(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header("subtitle", m.captures["c"]); + } + } else if (h_.match(regex(r"^@creator:|@author:"))) { + if (auto m = h_.match(regex(r"^(?:@creator:|@author:)(?:[ ]+(?P.+)|\n)"))) { + _tmp_header ~= format_main_header("creator", "author", m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pauthor):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + } else if (h_.match(regex(r"^@rights:"))) { + if (auto m = h_.match(regex(r"^@(?Prights):(?:[ ]+(?P.+)|\n)"))) { + _tmp_header ~= format_main_header(m.captures["h"], "copyright", m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pcopyright):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:licen[cs]e:(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header("license", m.captures["c"]); + } + } else if (h_.match(regex(r"^@date:|@date\."))) { + if (auto m = h_.match(regex(r"^@(?Pdate):(?:[ ]+(?P.+)|\n)"))) { + _tmp_header ~= format_main_header(m.captures["h"], "published", m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Ppublished):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pavailable):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pmodified):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pcreated):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pissued):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pvalid):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^@date\.(?Pavailable):[ ]+(?P.+)$"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^@date\.(?Pmodified):[ ]+(?P.+)$"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^@date\.(?Pcreated):[ ]+(?P.+)$"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^@date\.(?Pissued):[ ]+(?P.+)$"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^@date\.(?Pvalid):[ ]+(?P.+)$"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + } else if (h_.match(regex(r"^@classify:"))) { + if (auto m = h_.match(regex(r"^@classify:"))) { + _tmp_header ~= "classify:\n"; + } + if (auto m = h_.match(regex(r"^\s+:(?Ptopic_register):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:type:(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= "# type: " ~ "\"" ~ m.captures["c"] ~ "\"\n"; + } + } else if (h_.match(regex(r"^(?:@identifier:|@identify:)"))) { + if (auto m = h_.match(regex(r"^(?:@identifier:|@idenfify)"))) { + _tmp_header ~= "identify:\n"; + } + if (auto m = h_.match(regex(r"^\s+:(?Poclc):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pisbn):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pdewey):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + } else if (h_.match(regex(r"^@publisher:"))) { + if (auto m = h_.match(regex(r"^@publisher:[ ]+(?P.+)$"))) { + _tmp_header ~= "publisher: " ~ "\"" ~ m.captures["c"] ~ "\"\n"; + } + } else if (h_.match(regex(r"^@make:"))) { + // writeln(h_); + if (auto m = h_.match(regex(r"^@make:"))) { + _tmp_header ~= "make:\n"; + } + if (auto m = h_.match(regex(r"^\s+:(?Pbreaks):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pnum_top):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pheadings):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pitalics):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pbold):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pemphasis):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Ptexpdf_font):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Phome_button_text):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Phome_button_image):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pcover_image):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + if (auto m = h_.match(regex(r"^\s+:(?Pfooter):(?:[ ]+(?P.+)|$)", "m"))) { + _tmp_header ~= format_sub_header(m.captures["h"], m.captures["c"]); + } + // writeln(_tmp_header); + } else if (h_.match(regex(r"^@\w+:"))) { + _tmp_header ~= "# " ~ h_.split("\n").join("\n# ") ~ "\n"; + } else if (h_.match(regex(r"^\s+:\w+:", "m"))) { + if (auto m = h_.match(regex(r"^(?P\s+:\w+:.*)"))) { + _tmp_header ~= "# " ~ m.captures["g"] ~ "\n"; + } + } + if (h_.match(regex(r"^#", "m"))) { + if (auto m = h_.match(regex(r"^(?P#.*)", "m"))) { + _tmp_header ~= m.captures["g"] ~ "\n"; + } + } + if (_tmp_header.length > 0) { + munged_header ~= _tmp_header.split("\n\n"); + } else if (h_.length > 0) { + writeln("munging required: ", h_); + h_ = h_.replaceAll((regex(r"\n\n\n+", "m")), "\n\n"); + munged_header ~= h_; + } + } + // writeln(munged_header.join("\n")); + foreach (paragraph; paragraphs) { /+ loop to gather binary endnotes +/ + if ( !( type["curly_code"] == 1 || type["tic_code"] == 1) + && paragraph.match(rgx_endnote) + ) { + munged_endnotes ~= replaceAll!(m => m[1]) + (paragraph, rgx_endnote); + } else { + if ( type["curly_code"] == 1 || type["tic_code"] == 1 + || paragraph.matchFirst(block_curly_code_open) + || paragraph.matchFirst(block_tic_code_open) + ) { /+ code blocks identified, no munging +/ + if ( type["curly_code"] == 1 + && paragraph.matchFirst(block_curly_code_close) + ) { + type["curly_code"] = 0; + } else if (type["tic_code"] == 1 + && paragraph.matchFirst(block_tic_close) + ) { + type["tic_code"] = 0; + } else if (paragraph.matchFirst(block_curly_code_open)) { + type["curly_code"] = 1; + } else if (paragraph.matchFirst(block_tic_code_open)) { + type["tic_code"] = 1; + } + munged_contents ~= paragraph; + } else { /+ regular content, not a code block +/ + if (auto m = paragraph.matchAll(rgx_endnote_ref)) { + foreach (n; m) { + endnote_ref_count++; // endnote_refs ~= (n.captures[1]); + } + } + paragraph = format_body_string(paragraph); + // paragraph = replaceAll!(m => " \\\\ " ) + // (paragraph, regex(r"\s*<(?:/\s*|:)?br>\s*")); // (paragraph, regex(r"(
)")); + munged_contents ~= paragraph; + } + } + } + { + import std.outbuffer; + auto buffer = new OutBuffer(); + foreach (header; munged_header) { /+ loop to inline endnotes +/ + buffer.write(header ~ "\n"); + } + if (munged_endnotes.length == endnote_ref_count) { + int endnote_count = -1; + foreach (content; munged_contents) { /+ loop to inline endnotes +/ + content = replaceAll!(m => "~{ " ~ munged_endnotes[++endnote_count] ~ " }~" ~ m["tail"] ) + (content, rgx_endnote_ref); // endnote_ref cannot occur in a code block or else fail + buffer.write(content ~ "\n\n"); + } + if (buffer) { + try { + string dir_out = "converted_output_"; + string path_and_file_out = dir_out ~ "/" ~ filename; + dir_out.mkdirRecurse; + auto f = File(path_and_file_out, "w"); + f.write(buffer); + // writeln("wrote: ", path_and_file_out); + } catch (FileException ex) { + writeln("did not write file"); + // Handle errors + } + } + } else { + foreach (content; munged_contents) { /+ loop to inline endnotes +/ + buffer.write(content ~ "\n\n"); + } + } + } + } catch (ErrnoException ex) { + switch(ex.errno) { + case EPERM: + case EACCES: // Permission denied + break; + case ENOENT: // File does not exist + break; + default: // Handle other errors + break; + } + } + } + } +} -- cgit v1.2.3