From db92f9af3293f8009543448a100b08b7a2f2ffbe Mon Sep 17 00:00:00 2001
From: Ralph Amissah <ralph@amissah.com>
Date: Mon, 23 Apr 2018 12:07:22 -0400
Subject: 0.26.1 single sqlite file, premature?
---
src/sdp/meta/rgx.d | 2 +-
src/sdp/output/hub.d | 35 +-
src/sdp/output/paths_source.d | 54 +-
src/sdp/output/rgx.d | 2 +-
src/sdp/output/sqlite.d | 1930 +++++++++++++++++++-------------------
src/sdp/output/sqlite_discrete.d | 1023 --------------------
6 files changed, 1070 insertions(+), 1976 deletions(-)
delete mode 100644 src/sdp/output/sqlite_discrete.d
(limited to 'src/sdp')
diff --git a/src/sdp/meta/rgx.d b/src/sdp/meta/rgx.d
index 0f60847..6006122 100644
--- a/src/sdp/meta/rgx.d
+++ b/src/sdp/meta/rgx.d
@@ -199,7 +199,7 @@ static template SiSUrgxInit() {
static nbsp_and_space = ctRegex!(` [ ]`, "mg");
static nbsp_char_and_space = ctRegex!(`░[ ]`, "mg");
static special_markup_chars = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg");
- static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);
+ static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);
static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);
static src_pth_contents = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/sisupod[.]manifest$`);
static src_pth_pod_root = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`);
diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d
index 373092b..d395269 100644
--- a/src/sdp/output/hub.d
+++ b/src/sdp/output/hub.d
@@ -8,7 +8,6 @@ template outputHub() {
sdp.output.epub3,
sdp.output.html,
sdp.output.sqlite,
- sdp.output.sqlite_discrete,
sdp.output.xmls,
sdp.output.source_sisupod,
sdp.output.create_zip_file,
@@ -16,6 +15,35 @@ template outputHub() {
void outputHub(D,I)(D doc_abstraction, I doc_matters) {
mixin SiSUoutputRgxInit;
static auto rgx = Rgx();
+ debug (substitutions) {
+ enum Substitute { match, markup, html, }
+ writeln(__LINE__, ":", __FILE__, ": DEBUG substitutions:");
+ writeln("Doc Title: ", doc_matters.conf_make_meta.meta.title_full);
+ if (doc_matters.conf_make_meta.make.bold) {
+ writeln("regex to match: ", doc_matters.conf_make_meta.make.bold[Substitute.match]);
+ writeln("substitution to make: ", doc_matters.conf_make_meta.make.bold[Substitute.html]);
+
+ auto _w = "1. Debian test string. Debian again. (the problem) do not use initialized only not repopulated"
+ .replaceAll(
+ regex(doc_matters.conf_make_meta.make.bold[Substitute.match]),
+ doc_matters.conf_make_meta.make.bold[Substitute.html]
+ );
+ writeln(_w);
+
+ writeln("2a. Debian test string. Debian again. (the problem) do not use initialized only not repopulated"
+ .replaceAll(
+ regex(doc_matters.conf_make_meta.make.bold[Substitute.match]),
+ doc_matters.conf_make_meta.make.bold[Substitute.html]
+ ));
+
+ auto _v = regex(doc_matters.conf_make_meta.make.bold[Substitute.match]);
+ writeln("2b. Debian test string. Debian again. (the problem) do not use initialized only not repopulated"
+ .replaceAll(
+ _v,
+ doc_matters.conf_make_meta.make.bold[Substitute.html]
+ ));
+ }
+ }
if (doc_matters.opt.action.verbose) {
writeln(doc_matters.xml.keys_seq.seg);
}
@@ -109,13 +137,13 @@ template outputHub() {
if ((doc_matters.opt.action.verbose)) {
writeln("sqlite processing... ");
}
- SQLiteDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
+ SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
}
if (doc_matters.opt.action.sqlite_update) {
if ((doc_matters.opt.action.verbose)) {
writeln("sqlite processing... ");
}
- SQLiteBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
+ SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters);
}
if (doc_matters.opt.action.postgresql) {
/+ mixin outputPostgreSQL; +/
@@ -128,7 +156,6 @@ template outputHubOp() {
sdp.output.epub3,
sdp.output.html,
sdp.output.sqlite,
- sdp.output.sqlite_discrete,
sdp.output.xmls,
sdp.output.source_sisupod,
sdp.output.create_zip_file,
diff --git a/src/sdp/output/paths_source.d b/src/sdp/output/paths_source.d
index c420d7e..d04c9f6 100644
--- a/src/sdp/output/paths_source.d
+++ b/src/sdp/output/paths_source.d
@@ -151,6 +151,7 @@ template PathMatters() {
return Pod_();
}
auto src() {
+ string _sep = "␣";
struct SRC_ {
auto is_pod() {
auto _src_is_pod = (_manifest.pod_manifest_path.length > 0) ? true : false;
@@ -159,19 +160,70 @@ template PathMatters() {
auto path_and_fn() {
return _fns;
}
+ auto pod_name() { /+ work on +/
+ auto _pod_name = (is_pod)
+ ? _manifest.pod_manifest_path
+ : "";
+ return _pod_name;
+ }
auto filename() {
auto _fn = (path_and_fn).baseName;
return _fn;
}
+ auto filename_base() {
+ auto _fn = filename.stripExtension;
+ return _fn;
+ }
+ auto filename_extension() {
+ auto _ext = filename.match(rgx.src_pth_sst_or_ssm).captures["extension"];
+ return _ext;
+ }
auto lng() {
string _k;
- if (auto m = (path_and_fn).match(rgx.language_code_and_filename)) {
+ if (auto m = path_and_fn.match(rgx.language_code_and_filename)) {
_k = m.captures[1];
} else {
_k = "en";
}
return _k;
}
+ auto docname_composite_unique_per_src_doc() {
+ /+
+ z pod name if any + src filename + lng code
+ filename ~ "." ~ lng
+ * unique per src doc
+ used by
+ - sqlite discrete index (multilingual, each language of a document)
+ +/
+ string _fn;
+ if (pod_name.baseName == filename_base) {
+ _fn = filename_base ~ _sep ~ filename_extension ~ _sep ~ lng;
+ } else if (!(pod_name.empty)) {
+ _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng;
+ } else {
+ _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng;
+ }
+ return _fn;
+ }
+ auto docname_composite_unique_per_src_pod() {
+ /+
+ z pod name if any + src filename (without lng code)
+ filename ~ _sep ~ lng
+ * unique per src pod
+ used by
+ - sisupod (multilingual collection)
+ - sqlite discrete index (multilingual collection)
+ +/
+ string _fn;
+ if (pod_name.baseName == filename_base) {
+ _fn = filename_base ~ _sep ~ filename_extension;
+ } else if (!(pod_name.empty)) {
+ _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension;
+ } else {
+ _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension;
+ }
+ return _fn;
+ }
auto language() {
return lng();
}
diff --git a/src/sdp/output/rgx.d b/src/sdp/output/rgx.d
index d559491..4bfa72a 100644
--- a/src/sdp/output/rgx.d
+++ b/src/sdp/output/rgx.d
@@ -16,7 +16,7 @@ static template SiSUoutputRgxInit() {
static nbsp_and_space = ctRegex!(` [ ]`, "mg");
static nbsp_char_and_space = ctRegex!(`░[ ]`, "mg");
static special_markup_chars = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg");
- static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);
+ static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);
static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])$`);
static src_pth_contents = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+)/sisupod[.]manifest$`);
static src_pth_pod_root = ctRegex!(`^(?P<podroot>(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`);
diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d
index 2409e3b..fc1843f 100644
--- a/src/sdp/output/sqlite.d
+++ b/src/sdp/output/sqlite.d
@@ -3,7 +3,79 @@ import sdp.output;
import
std.file,
std.conv : to;
+import d2sqlite3;
+import std.typecons : Nullable;
+mixin SiSUoutputRgxInit;
+static auto rgx = Rgx();
long _metadata_tid_lastrowid;
+template SQLiteHubBuildTablesAndPopulate() {
+ void SQLiteHubBuildTablesAndPopulate(D,I)(
+ auto ref const D doc_abstraction,
+ auto ref I doc_matters,
+ ) {
+ auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
+ pth_sqlite.base.mkdirRecurse;
+ auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName));
+ template SQLiteDbStatementComposite() {
+ void SQLiteDbStatementComposite(Db,D,I)(
+ auto ref Db db,
+ auto ref const D doc_abstraction,
+ auto ref I doc_matters,
+ ) {
+ {
+ string _db_statement;
+ {
+ if ((doc_matters.opt.action.sqlite_create)) {
+ auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
+ pth_sqlite.base.mkdirRecurse;
+ _db_statement ~= SQLiteTablesReCreate!()();
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE");
+ }
+ if ((doc_matters.opt.action.sqlite_update)) { // TODO
+ _db_statement ~= SQLiteInsertMetadata!()(doc_matters);
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData");
+ /+ get tid (lastrowid or max) for use in doc_objects table +/
+ _metadata_tid_lastrowid = db.lastInsertRowid();
+ _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects");
+ }
+ }
+ db.close;
+ }
+ }
+ }
+ SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);
+ }
+}
+template SQLiteHubDiscreteBuildTablesAndPopulate() {
+ void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)(
+ auto ref const D doc_abstraction,
+ auto ref I doc_matters,
+ ) {
+ auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language);
+ pth_sqlite.base.mkdirRecurse;
+ auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename));
+ template SQLiteDiscreteDbStatementComposite() {
+ void SQLiteDiscreteDbStatementComposite(Db,D,I)(
+ auto ref Db db,
+ auto ref const D doc_abstraction,
+ auto ref I doc_matters,
+ ) {
+ {
+ string _db_statement;
+ {
+ _db_statement ~= SQLiteTablesReCreate!()();
+ _db_statement ~= SQLiteInsertMetadata!()(doc_matters);
+ _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX
+ SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects");
+ }
+ db.close;
+ }
+ }
+ }
+ SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);
+ }
+}
template SQLiteDbRun() {
void SQLiteDbRun(Db,St,O)(
auto ref Db db,
@@ -20,7 +92,9 @@ template SQLiteDbRun() {
db_statement ~
"\nCOMMIT;\n"
);
- if (!(opt_action.sqlite_create)) {
+ if (!(opt_action.sqlite_discrete)
+ && !(opt_action.sqlite_create)
+ ) {
_metadata_tid_lastrowid = db.lastInsertRowid();
writeln("last row id: ", _metadata_tid_lastrowid);
}
@@ -37,1005 +111,968 @@ template SQLiteDbRun() {
}
}
}
-template SQLiteBuildTablesAndPopulate() {
- void SQLiteBuildTablesAndPopulate(D,I)(
- auto ref const D doc_abstraction,
+template SQLinsertDelimiter() {
+ auto SQLinsertDelimiter(string _txt) {
+ _txt = _txt
+ .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0");
+ return _txt;
+ }
+}
+template SQLiteFormatAndLoadObject() {
+ auto SQLiteFormatAndLoadObject(I)(
auto ref I doc_matters,
) {
- import d2sqlite3;
- import std.typecons : Nullable;
mixin SiSUoutputRgxInit;
- static auto rgx = Rgx();
- auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path);
- pth_sqlite.base.mkdirRecurse;
- auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName));
- template SQLiteDbStatementComposite() {
- void SQLiteDbStatementComposite(Db,D,I)(
- auto ref Db db,
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
+ struct sqlite_format_and_load_objects {
+ auto generic_munge_sanitize_text_for_search(
+ string _txt,
) {
- {
- {
- string _db_statement;
- {
- if ((doc_matters.opt.action.sqlite_create)) {
- auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language);
- pth_sqlite.base.mkdirRecurse;
- _db_statement ~= SQLiteTablesReCreate!()();
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE");
- }
- if ((doc_matters.opt.action.sqlite_update)) { // TODO
- _db_statement ~= SQLiteInsertMetadata!()(doc_matters);
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData");
- /+ get tid (lastrowid or max) for use in doc_objects table +/
- _metadata_tid_lastrowid = db.lastInsertRowid();
- _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX
- SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects");
- }
+ string _notes;
+ string _urls;
+ if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
+ foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
+ _notes ~= "\n" ~ m["text"];
+ }
+ _txt = _txt.replaceAll(rgx.inline_notes_al_gen, "");
+ }
+ if (_txt.matchFirst(rgx.inline_link)) {
+ foreach (m; _txt.matchAll(rgx.inline_link)) {
+ if (m["link"].match(rgx.url)) {
+ _urls ~= "\n" ~ m["link"];
}
- db.close;
}
+ _txt = _txt.replaceAll(rgx.inline_link_clean, "");
+ }
+ if (_notes.length > 0) {
+ _txt ~= _notes;
+ }
+ if (_urls.length > 0) {
+ _txt ~= _urls;
}
+ if (doc_matters.opt.action.debug_do) {
+ writeln(_txt, "\n");
+ }
+ debug(sql_text_clean) {
+ writeln(_txt);
+ }
+ return _txt;
}
- }
- template SQLinsertDelimiter() {
- auto SQLinsertDelimiter(string _txt) {
+ auto munge_html(O)(
+ auto return ref const O obj,
+ ) {
+ string _html_special_characters(string _txt){
+ _txt = _txt
+ .replaceAll(rgx.xhtml_ampersand, "&")
+ .replaceAll(rgx.xhtml_quotation, """)
+ .replaceAll(rgx.xhtml_less_than, "<")
+ .replaceAll(rgx.xhtml_greater_than, ">")
+ .replaceAll(rgx.nbsp_char, " ")
+ .replaceAll(rgx.xhtml_line_break, "<br />");
+ return _txt;
+ }
+ string _html_font_face(string _txt){
+ _txt = _txt
+ .replaceAll(rgx.inline_emphasis, "<em>$1</em>")
+ .replaceAll(rgx.inline_bold, "<b>$1</b>")
+ .replaceAll(rgx.inline_underscore, "<u>$1</u>")
+ .replaceAll(rgx.inline_italics, "<i>$1</i>")
+ .replaceAll(rgx.inline_superscript, "<sup>$1</sup>")
+ .replaceAll(rgx.inline_subscript, "<sub>$1</sub>")
+ .replaceAll(rgx.inline_strike, "<del>$1</del>")
+ .replaceAll(rgx.inline_insert, "<ins>$1</ins>")
+ .replaceAll(rgx.inline_mono, "<tt>$1</tt>")
+ .replaceAll(rgx.inline_cite, "<cite>$1</cite>");
+ return _txt;
+ }
+ string _notes;
+ string _urls;
+ string _txt = _html_font_face(_html_special_characters(obj.text));
+ if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
+ foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
+ _notes ~= "\n" ~ m["text"];
+ }
+ _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> ");
+ }
+ if (_txt.matchFirst(rgx.inline_link)) {
+ foreach (m; _txt.matchAll(rgx.inline_link)) {
+ }
+ _txt = _txt.replaceAll(rgx.inline_link_clean, "");
+ }
+ if (_notes.length > 0) {
+ _txt ~= _notes;
+ }
+ if (doc_matters.opt.action.debug_do) {
+ writeln(_txt, "\n");
+ }
+ return _txt;
+ }
+ string html_special_characters(string _txt){
_txt = _txt
- .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0");
+ .replaceAll(rgx.xhtml_ampersand, "&")
+ .replaceAll(rgx.xhtml_quotation, """)
+ .replaceAll(rgx.xhtml_less_than, "<")
+ .replaceAll(rgx.xhtml_greater_than, ">")
+ .replaceAll(rgx.nbsp_char, " ")
+ .replaceAll(rgx.xhtml_line_break, "<br />");
return _txt;
}
- }
- template SQLiteFormatAndLoadObject() {
- auto SQLiteFormatAndLoadObject(I)(
- auto ref I doc_matters,
+ string html_special_characters_code(string _txt){
+ _txt = _txt
+ .replaceAll(rgx.xhtml_ampersand, "&")
+ .replaceAll(rgx.xhtml_quotation, """)
+ .replaceAll(rgx.xhtml_less_than, "<")
+ .replaceAll(rgx.xhtml_greater_than, ">")
+ .replaceAll(rgx.nbsp_char, " ");
+ return _txt;
+ }
+ string html_font_face(string _txt){
+ _txt = _txt
+ .replaceAll(rgx.inline_emphasis, "<em>$1</em>")
+ .replaceAll(rgx.inline_bold, "<b>$1</b>")
+ .replaceAll(rgx.inline_underscore, "<u>$1</u>")
+ .replaceAll(rgx.inline_italics, "<i>$1</i>")
+ .replaceAll(rgx.inline_superscript, "<sup>$1</sup>")
+ .replaceAll(rgx.inline_subscript, "<sub>$1</sub>")
+ .replaceAll(rgx.inline_strike, "<del>$1</del>")
+ .replaceAll(rgx.inline_insert, "<ins>$1</ins>")
+ .replaceAll(rgx.inline_mono, "<tt>$1</tt>")
+ .replaceAll(rgx.inline_cite, "<cite>$1</cite>");
+ return _txt;
+ }
+ auto html_heading(O)(
+ auto return ref const O obj,
) {
- mixin SiSUoutputRgxInit;
- struct sqlite_format_and_load_objects {
- auto generic_munge_sanitize_text_for_search(
- string _txt,
- ) {
- string _notes;
- string _urls;
- if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
- foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
- _notes ~= "\n" ~ m["text"];
- }
- _txt = _txt.replaceAll(rgx.inline_notes_al_gen, "");
- }
- if (_txt.matchFirst(rgx.inline_link)) {
- foreach (m; _txt.matchAll(rgx.inline_link)) {
- if (m["link"].match(rgx.url)) {
- _urls ~= "\n" ~ m["link"];
+ string _txt = munge_html(obj);
+ string o = format(q"¶<p class="%s"><b>
+ %s
+ </b></p>¶",
+ obj.is_a,
+ _txt,
+ );
+ return o;
+ }
+ auto html_para(O)(
+ auto return ref const O obj,
+ ) {
+ string _txt = munge_html(obj);
+ _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt;
+ string o = format(q"¶<p class="%s" indent="h%si%s">
+ %s
+ </p>¶",
+ obj.is_a,
+ obj.indent_hang,
+ obj.indent_base,
+ _txt
+ );
+ return o;
+ }
+ auto html_quote(O)(
+ auto return ref const O obj,
+ ) {
+ string _txt = munge_html(obj);
+ string o = format(q"¶<p class="%s">
+ %s
+ </p>¶",
+ obj.is_a,
+ _txt
+ );
+ return o;
+ }
+ auto html_group(O)(
+ auto return ref const O obj,
+ ) {
+ string _txt = munge_html(obj);
+ string o = format(q"¶<p class="%s">
+ %s
+ </p>¶",
+ obj.is_a,
+ _txt
+ );
+ return o;
+ }
+ auto html_block(O)(
+ auto return ref const O obj,
+ ) {
+ string _txt = munge_html(obj);
+ string o = format(q"¶
+ <p class="%s">%s</p>¶",
+ obj.is_a,
+ _txt.stripRight
+ );
+ return o;
+ }
+ auto html_verse(O)(
+ auto return ref const O obj,
+ ) {
+ string _txt = munge_html(obj);
+ string o = format(q"¶<p class="%s">%s</p>¶",
+ obj.is_a,
+ _txt
+ );
+ return o;
+ }
+ auto html_code(O)(
+ auto return ref const O obj,
+ ) {
+ string _txt = html_special_characters_code(obj.text);
+ string o = format(q"¶<p class="%s">%s</p>¶",
+ obj.is_a,
+ _txt
+ );
+ return o;
+ }
+ auto html_table(O)(
+ auto return ref const O obj,
+ ) {
+ auto _tablarize(O)(
+ auto return ref const O obj,
+ string _txt,
+ ) {
+ string[] _table_rows = _txt.split(rgx.table_delimiter_row);
+ string[] _table_cols;
+ string _table;
+ string _tablenote;
+ foreach(row_idx, row; _table_rows) {
+ _table_cols = row.split(rgx.table_delimiter_col);
+ _table ~= "<tr>";
+ foreach(col_idx, cell; _table_cols) {
+ if ((_table_cols.length == 1)
+ && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx)
+ _tablenote ~= cell;
+ } else {
+ string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td";
+ string _align = ("style=\"text-align:"
+ ~ ((obj.table_column_aligns[col_idx] == "l")
+ ? "left\"" : "right\""));
+ _table ~= "<"
+ ~ _col_is
+ ~ " width=\""
+ ~ obj.table_column_widths[col_idx].to!string
+ ~ "%\" "
+ ~ _align
+ ~ ">";
+ _table ~= cell;
+ _table ~= "</"
+ ~ _col_is
+ ~ ">";
}
}
- _txt = _txt.replaceAll(rgx.inline_link_clean, "");
- }
- if (_notes.length > 0) {
- _txt ~= _notes;
- }
- if (_urls.length > 0) {
- _txt ~= _urls;
- }
- if (doc_matters.opt.action.debug_do) {
- writeln(_txt, "\n");
- }
- debug(sql_text_clean) {
- writeln(_txt);
- }
- return _txt;
- }
- auto munge_html(O)(
- auto return ref const O obj,
- ) {
- string _html_special_characters(string _txt){
- _txt = _txt
- .replaceAll(rgx.xhtml_ampersand, "&")
- .replaceAll(rgx.xhtml_quotation, """)
- .replaceAll(rgx.xhtml_less_than, "<")
- .replaceAll(rgx.xhtml_greater_than, ">")
- .replaceAll(rgx.nbsp_char, " ")
- .replaceAll(rgx.xhtml_line_break, "<br />");
- return _txt;
- }
- string _html_font_face(string _txt){
- _txt = _txt
- .replaceAll(rgx.inline_emphasis, "<em>$1</em>")
- .replaceAll(rgx.inline_bold, "<b>$1</b>")
- .replaceAll(rgx.inline_underscore, "<u>$1</u>")
- .replaceAll(rgx.inline_italics, "<i>$1</i>")
- .replaceAll(rgx.inline_superscript, "<sup>$1</sup>")
- .replaceAll(rgx.inline_subscript, "<sub>$1</sub>")
- .replaceAll(rgx.inline_strike, "<del>$1</del>")
- .replaceAll(rgx.inline_insert, "<ins>$1</ins>")
- .replaceAll(rgx.inline_mono, "<tt>$1</tt>")
- .replaceAll(rgx.inline_cite, "<cite>$1</cite>");
- return _txt;
+ _table ~= "</tr>";
}
- string _notes;
- string _urls;
- string _txt = _html_font_face(_html_special_characters(obj.text));
- if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
- foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
- _notes ~= "\n" ~ m["text"];
- }
- _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> ");
- }
- if (_txt.matchFirst(rgx.inline_link)) {
- foreach (m; _txt.matchAll(rgx.inline_link)) {
- }
- _txt = _txt.replaceAll(rgx.inline_link_clean, "");
- }
- if (_notes.length > 0) {
- _txt ~= _notes;
- }
- if (doc_matters.opt.action.debug_do) {
- writeln(_txt, "\n");
- }
- return _txt;
- }
- string html_special_characters(string _txt){
- _txt = _txt
- .replaceAll(rgx.xhtml_ampersand, "&")
- .replaceAll(rgx.xhtml_quotation, """)
- .replaceAll(rgx.xhtml_less_than, "<")
- .replaceAll(rgx.xhtml_greater_than, ">")
- .replaceAll(rgx.nbsp_char, " ")
- .replaceAll(rgx.xhtml_line_break, "<br />");
- return _txt;
- }
- string html_special_characters_code(string _txt){
- _txt = _txt
- .replaceAll(rgx.xhtml_ampersand, "&")
- .replaceAll(rgx.xhtml_quotation, """)
- .replaceAll(rgx.xhtml_less_than, "<")
- .replaceAll(rgx.xhtml_greater_than, ">")
- .replaceAll(rgx.nbsp_char, " ");
- return _txt;
- }
- string html_font_face(string _txt){
- _txt = _txt
- .replaceAll(rgx.inline_emphasis, "<em>$1</em>")
- .replaceAll(rgx.inline_bold, "<b>$1</b>")
- .replaceAll(rgx.inline_underscore, "<u>$1</u>")
- .replaceAll(rgx.inline_italics, "<i>$1</i>")
- .replaceAll(rgx.inline_superscript, "<sup>$1</sup>")
- .replaceAll(rgx.inline_subscript, "<sub>$1</sub>")
- .replaceAll(rgx.inline_strike, "<del>$1</del>")
- .replaceAll(rgx.inline_insert, "<ins>$1</ins>")
- .replaceAll(rgx.inline_mono, "<tt>$1</tt>")
- .replaceAll(rgx.inline_cite, "<cite>$1</cite>");
- return _txt;
- }
- auto html_heading(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s"><b>
- %s
- </b></p>¶",
- obj.is_a,
- _txt,
- );
- return o;
- }
- auto html_para(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt;
- string o = format(q"¶<p class="%s" indent="h%si%s">
- %s
- </p>¶",
- obj.is_a,
- obj.indent_hang,
- obj.indent_base,
- _txt
- );
- return o;
- }
- auto html_quote(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s">
- %s
- </p>¶",
- obj.is_a,
- _txt
- );
- return o;
- }
- auto html_group(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s">
- %s
- </p>¶",
- obj.is_a,
- _txt
- );
- return o;
+ auto t = tuple(
+ _table,
+ _tablenote,
+ );
+ return t;
+ }
+ string _txt = munge_html(obj);
+ auto t = _tablarize(obj, _txt);
+ _txt = t[0];
+ string _note = t[1];
+ string o = format(q"¶<p class="%s">
+ <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center">
+ %s
+ </table>
+ %s
+ </p>¶",
+ obj.is_a,
+ _txt,
+ _note
+ );
+ return o;
+ }
+ auto sqlite_load_string(O,Dm)(
+ auto return ref const O obj,
+ auto return ref Dm doc_matters,
+ ) {
+ string o;
+ return o;
+ }
+ auto postgresql_load_string(O,Dm)(
+ auto return ref const O obj,
+ auto return ref Dm doc_matters,
+ ) {
+ string o;
+ return o;
+ }
+ string sqlite_statement(O)(
+ auto return ref const O obj,
+ string _txt,
+ string _html,
+ ) {
+ void _sql_exe(O)(
+ string _sql,
+ ) {
+ writeln(_html);
+ writeln(_sql);
+ }
+ string _sql;
+ return _sql;
+ }
+ auto heading(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_heading(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto html_block(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶
- <p class="%s">%s</p>¶",
- obj.is_a,
- _txt.stripRight
- );
- return o;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto html_verse(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s">%s</p>¶",
- obj.is_a,
- _txt
- );
- return o;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto para(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_para(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto html_code(O)(
- auto return ref const O obj,
- ) {
- string _txt = html_special_characters_code(obj.text);
- string o = format(q"¶<p class="%s">%s</p>¶",
- obj.is_a,
- _txt
- );
- return o;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto html_table(O)(
- auto return ref const O obj,
- ) {
- auto _tablarize(O)(
- auto return ref const O obj,
- string _txt,
- ) {
- string[] _table_rows = _txt.split(rgx.table_delimiter_row);
- string[] _table_cols;
- string _table;
- string _tablenote;
- foreach(row_idx, row; _table_rows) {
- _table_cols = row.split(rgx.table_delimiter_col);
- _table ~= "<tr>";
- foreach(col_idx, cell; _table_cols) {
- if ((_table_cols.length == 1)
- && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx)
- _tablenote ~= cell;
- } else {
- string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td";
- string _align = ("style=\"text-align:"
- ~ ((obj.table_column_aligns[col_idx] == "l")
- ? "left\"" : "right\""));
- _table ~= "<"
- ~ _col_is
- ~ " width=\""
- ~ obj.table_column_widths[col_idx].to!string
- ~ "%\" "
- ~ _align
- ~ ">";
- _table ~= cell;
- _table ~= "</"
- ~ _col_is
- ~ ">";
- }
- }
- _table ~= "</tr>";
- }
- auto t = tuple(
- _table,
- _tablenote,
- );
- return t;
- }
- string _txt = munge_html(obj);
- auto t = _tablarize(obj, _txt);
- _txt = t[0];
- string _note = t[1];
- string o = format(q"¶<p class="%s">
- <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center">
- %s
- </table>
- %s
- </p>¶",
- obj.is_a,
- _txt,
- _note
- );
- return o;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto quote(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_quote(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto sqlite_load_string(O,Dm)(
- auto return ref const O obj,
- auto return ref Dm doc_matters,
- ) {
- string o;
- return o;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto postgresql_load_string(O,Dm)(
- auto return ref const O obj,
- auto return ref Dm doc_matters,
- ) {
- string o;
- return o;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto group(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_group(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- string sqlite_statement(O)(
- auto return ref const O obj,
- string _txt,
- string _html,
- ) {
- void _sql_exe(O)(
- string _sql,
- ) {
- writeln(_html);
- writeln(_sql);
- }
- string _sql;
- return _sql;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto heading(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_heading(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto block(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_block(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto para(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_para(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto quote(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_quote(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto verse(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_verse(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto group(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_group(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto block(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_block(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto code(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_code(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto verse(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_verse(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
- auto code(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_code(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ } else {
+ // load sql
+ }
+ return obj_txt;
+ }
+ auto table(O)(
+ auto return ref const O obj,
+ ) {
+ string[string] obj_txt = [
+ "text": generic_munge_sanitize_text_for_search(obj.text),
+ "html": html_table(obj)
+ ];
+ if (doc_matters.opt.action.debug_do) {
+ debug(sql_txt) {
+ writeln(obj_txt["text"]);
}
- auto table(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_table(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
+ debug(sql_html) {
+ writeln(obj_txt["html"]);
}
+ } else {
+ // load sql
}
- return sqlite_format_and_load_objects();
+ return obj_txt;
}
}
- template SQLiteTablesReCreate() {
- string SQLiteTablesReCreate()() {
- string _sql_instruct;
- _sql_instruct = format(q"¶
- DROP INDEX IF EXISTS ocn;
- DROP INDEX IF EXISTS digest_clean;
- DROP INDEX IF EXISTS digest_all;
- DROP INDEX IF EXISTS clean;
- DROP INDEX IF EXISTS lev0;
- DROP INDEX IF EXISTS lev1;
- DROP INDEX IF EXISTS lev2;
- DROP INDEX IF EXISTS lev3;
- DROP INDEX IF EXISTS lev4;
- DROP INDEX IF EXISTS lev5;
- DROP INDEX IF EXISTS lev6;
- DROP INDEX IF EXISTS title;
- DROP INDEX IF EXISTS creator_author;
- DROP INDEX IF EXISTS src_filename;
- DROP INDEX IF EXISTS language_document_char;
- DROP INDEX IF EXISTS classify_topic_register;
- DROP TABLE IF EXISTS metadata_and_text;
- DROP TABLE IF EXISTS doc_objects;
- DROP TABLE IF EXISTS urls;
- CREATE TABLE metadata_and_text (
- tid INTEGER PRIMARY KEY,
- /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
- title VARCHAR(800) NOT NULL,
- title_main VARCHAR(400) NOT NULL,
- title_sub VARCHAR(400) NULL,
- title_short VARCHAR(400) NULL,
- title_edition VARCHAR(10) NULL,
- title_note VARCHAR(2500) NULL,
- title_language VARCHAR(100) NULL,
- title_language_char VARCHAR(6) NULL,
- creator_author VARCHAR(600) NOT NULL,
- creator_author_email VARCHAR(100) NULL,
- creator_author_hon VARCHAR(100) NULL,
- creator_author_nationality VARCHAR(100) NULL,
- creator_editor VARCHAR(600) NULL,
- creator_contributor VARCHAR(600) NULL,
- creator_illustrator VARCHAR(600) NULL,
- creator_photographer VARCHAR(600) NULL,
- creator_translator VARCHAR(600) NULL,
- creator_prepared_by VARCHAR(600) NULL,
- creator_digitized_by VARCHAR(600) NULL,
- creator_audio VARCHAR(600) NULL,
- creator_video VARCHAR(600) NULL,
- language_document VARCHAR(100) NULL,
- language_document_char VARCHAR(6) NOT NULL,
- language_original VARCHAR(100) NULL,
- language_original_char VARCHAR(6) NULL,
- date_added_to_site VARCHAR(10) NULL,
- date_available VARCHAR(10) NULL,
- date_created VARCHAR(10) NULL,
- date_issued VARCHAR(10) NULL,
- date_modified VARCHAR(10) NULL,
- date_published VARCHAR(10) NULL,
- date_valid VARCHAR(10) NULL,
- date_translated VARCHAR(10) NULL,
- date_original_publication VARCHAR(10) NULL,
- date_generated VARCHAR(10) NULL,
- publisher VARCHAR(600) NULL,
- original_publisher VARCHAR(600) NULL,
- original_language VARCHAR(100) NULL,
- original_language_char VARCHAR(6) NULL,
- original_source VARCHAR(600) NULL,
- original_institution VARCHAR(600) NULL,
- original_nationality VARCHAR(100) NULL,
- original_title VARCHAR(800) NULL,
- rights_copyright VARCHAR(2500) NULL,
- rights_copyright_audio VARCHAR(2500) NULL,
- rights_copyright_cover VARCHAR(2500) NULL,
- rights_copyright_illustrations VARCHAR(2500) NULL,
- rights_copyright_photographs VARCHAR(2500) NULL,
- rights_copyright_text VARCHAR(2500) NULL,
- rights_copyright_translation VARCHAR(2500) NULL,
- rights_copyright_video VARCHAR(2500) NULL,
- rights_license VARCHAR(2500) NULL,
- identifier_oclc VARCHAR(30) NULL,
- identifier_isbn VARCHAR(16) NULL,
- classify_topic_register VARCHAR(2500) NULL,
- classify_subject VARCHAR(600) NULL,
- classify_loc VARCHAR(30) NULL,
- classify_dewey VARCHAR(30) NULL,
- classify_keywords VARCHAR(600) NULL,
- notes_abstract TEXT NULL,
- notes_description TEXT NULL,
- notes_comment TEXT NULL,
- notes_coverage VARCHAR(200) NULL,
- notes_relation VARCHAR(200) NULL,
- notes_history VARCHAR(600) NULL,
- notes_type VARCHAR(600) NULL,
- notes_format VARCHAR(600) NULL,
- notes_prefix TEXT NULL,
- notes_prefix_a TEXT NULL,
- notes_prefix_b TEXT NULL,
- notes_suffix TEXT NULL,
- src_filename VARCHAR(256) NOT NULL,
- src_fingerprint VARCHAR(256) NULL,
- src_filesize VARCHAR(10) NULL,
- src_wordcount VARCHAR(10) NULL,
- pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */
- pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */
- pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */
- src_text TEXT NULL,
- fulltext TEXT NULL,
- links TEXT NULL
- );
- CREATE TABLE doc_objects (
- lid INTEGER PRIMARY KEY,
- metadata_tid INTEGER REFERENCES metadata_and_text,
- ocn SMALLINT,
- ocnd VARCHAR(6),
- ocns VARCHAR(6),
- clean TEXT NULL,
- body TEXT NULL,
- book_idx TEXT NULL,
- seg VARCHAR(256) NULL,
- lev_an VARCHAR(1),
- lev SMALLINT NULL,
- lev0 SMALLINT,
- lev1 SMALLINT,
- lev2 SMALLINT,
- lev3 SMALLINT,
- lev4 SMALLINT,
- lev5 SMALLINT,
- lev6 SMALLINT,
- lev7 SMALLINT,
- en_a SMALLINT NULL,
- en_z SMALLINT NULL,
- en_a_asterisk SMALLINT NULL,
- en_z_asterisk SMALLINT NULL,
- en_a_plus SMALLINT NULL,
- en_z_plus SMALLINT NULL,
- t_of VARCHAR(16),
- t_is VARCHAR(16),
- node VARCHAR(16) NULL,
- parent VARCHAR(16) NULL,
- digest_clean CHAR(256),
- digest_all CHAR(256),
- types CHAR(1) NULL
- );
- CREATE INDEX idx_ocn ON doc_objects(ocn);
- CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);
- CREATE INDEX idx_digest_all ON doc_objects(digest_all);
- CREATE INDEX idx_clean ON doc_objects(clean);
- CREATE INDEX idx_lev0 ON doc_objects(lev0);
- CREATE INDEX idx_lev1 ON doc_objects(lev1);
- CREATE INDEX idx_lev2 ON doc_objects(lev2);
- CREATE INDEX idx_lev3 ON doc_objects(lev3);
- CREATE INDEX idx_lev4 ON doc_objects(lev4);
- CREATE INDEX idx_lev5 ON doc_objects(lev5);
- CREATE INDEX idx_lev6 ON doc_objects(lev6);
- CREATE INDEX idx_title ON metadata_and_text(title);
- CREATE INDEX idx_author ON metadata_and_text(creator_author);
- CREATE INDEX idx_filename ON metadata_and_text(src_filename);
- CREATE INDEX idx_language ON metadata_and_text(language_document_char);
- CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);
- ¶",);
- return _sql_instruct;
- }
- }
- template SQLiteInsertMetadata() {
- string SQLiteInsertMetadata(I)(
- auto ref I doc_matters,
- ) {
- string _insert_metadata;
- _insert_metadata = format(q"¶
- INSERT INTO metadata_and_text (
- src_filename,
- title,
- title_main,
- title_sub,
- title_short,
- title_edition,
- title_language,
- classify_dewey,
- classify_keywords,
- classify_loc,
- classify_subject,
- classify_topic_register,
- creator_author,
- creator_author_email,
- creator_illustrator,
- creator_translator,
- date_added_to_site,
- date_available,
- date_created,
- date_issued,
- date_modified,
- date_published,
- date_valid,
- identifier_isbn,
- identifier_oclc,
- language_document,
- language_document_char,
- notes_abstract,
- notes_description,
- original_publisher,
- original_language,
- original_language_char,
- original_source,
- original_title,
- publisher,
- rights_copyright,
- rights_copyright_audio,
- rights_copyright_cover,
- rights_copyright_illustrations,
- rights_copyright_photographs,
- rights_copyright_text,
- rights_copyright_translation,
- rights_copyright_video,
- rights_license
- )
- VALUES (
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s'
- );
- ¶",
- SQLinsertDelimiter!()(doc_matters.src.filename),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license)
+ return sqlite_format_and_load_objects();
+ }
+}
+template SQLiteTablesReCreate() {
+ string SQLiteTablesReCreate()() {
+ string _sql_instruct;
+ _sql_instruct = format(q"¶
+ DROP INDEX IF EXISTS ocn;
+ DROP INDEX IF EXISTS digest_clean;
+ DROP INDEX IF EXISTS digest_all;
+ DROP INDEX IF EXISTS clean;
+ DROP INDEX IF EXISTS lev0;
+ DROP INDEX IF EXISTS lev1;
+ DROP INDEX IF EXISTS lev2;
+ DROP INDEX IF EXISTS lev3;
+ DROP INDEX IF EXISTS lev4;
+ DROP INDEX IF EXISTS lev5;
+ DROP INDEX IF EXISTS lev6;
+ DROP INDEX IF EXISTS title;
+ DROP INDEX IF EXISTS creator_author;
+ DROP INDEX IF EXISTS src_filename;
+ DROP INDEX IF EXISTS language_document_char;
+ DROP INDEX IF EXISTS classify_topic_register;
+ DROP TABLE IF EXISTS metadata_and_text;
+ DROP TABLE IF EXISTS doc_objects;
+ DROP TABLE IF EXISTS urls;
+ CREATE TABLE metadata_and_text (
+ tid INTEGER PRIMARY KEY,
+ src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ title VARCHAR(800) NOT NULL,
+ title_main VARCHAR(400) NOT NULL,
+ title_sub VARCHAR(400) NULL,
+ title_short VARCHAR(400) NULL,
+ title_edition VARCHAR(10) NULL,
+ title_note VARCHAR(2500) NULL,
+ title_language VARCHAR(100) NULL,
+ title_language_char VARCHAR(6) NULL,
+ creator_author VARCHAR(600) NOT NULL,
+ creator_author_email VARCHAR(100) NULL,
+ creator_author_hon VARCHAR(100) NULL,
+ creator_author_nationality VARCHAR(100) NULL,
+ creator_editor VARCHAR(600) NULL,
+ creator_contributor VARCHAR(600) NULL,
+ creator_illustrator VARCHAR(600) NULL,
+ creator_photographer VARCHAR(600) NULL,
+ creator_translator VARCHAR(600) NULL,
+ creator_prepared_by VARCHAR(600) NULL,
+ creator_digitized_by VARCHAR(600) NULL,
+ creator_audio VARCHAR(600) NULL,
+ creator_video VARCHAR(600) NULL,
+ language_document VARCHAR(100) NULL,
+ language_document_char VARCHAR(6) NOT NULL,
+ language_original VARCHAR(100) NULL,
+ language_original_char VARCHAR(6) NULL,
+ date_added_to_site VARCHAR(10) NULL,
+ date_available VARCHAR(10) NULL,
+ date_created VARCHAR(10) NULL,
+ date_issued VARCHAR(10) NULL,
+ date_modified VARCHAR(10) NULL,
+ date_published VARCHAR(10) NULL,
+ date_valid VARCHAR(10) NULL,
+ date_translated VARCHAR(10) NULL,
+ date_original_publication VARCHAR(10) NULL,
+ date_generated VARCHAR(10) NULL,
+ publisher VARCHAR(600) NULL,
+ original_publisher VARCHAR(600) NULL,
+ original_language VARCHAR(100) NULL,
+ original_language_char VARCHAR(6) NULL,
+ original_source VARCHAR(600) NULL,
+ original_institution VARCHAR(600) NULL,
+ original_nationality VARCHAR(100) NULL,
+ original_title VARCHAR(800) NULL,
+ rights_copyright VARCHAR(2500) NULL,
+ rights_copyright_audio VARCHAR(2500) NULL,
+ rights_copyright_cover VARCHAR(2500) NULL,
+ rights_copyright_illustrations VARCHAR(2500) NULL,
+ rights_copyright_photographs VARCHAR(2500) NULL,
+ rights_copyright_text VARCHAR(2500) NULL,
+ rights_copyright_translation VARCHAR(2500) NULL,
+ rights_copyright_video VARCHAR(2500) NULL,
+ rights_license VARCHAR(2500) NULL,
+ identifier_oclc VARCHAR(30) NULL,
+ identifier_isbn VARCHAR(16) NULL,
+ classify_topic_register VARCHAR(2500) NULL,
+ classify_subject VARCHAR(600) NULL,
+ classify_loc VARCHAR(30) NULL,
+ classify_dewey VARCHAR(30) NULL,
+ classify_keywords VARCHAR(600) NULL,
+ notes_abstract TEXT NULL,
+ notes_description TEXT NULL,
+ notes_comment TEXT NULL,
+ notes_coverage VARCHAR(200) NULL,
+ notes_relation VARCHAR(200) NULL,
+ notes_history VARCHAR(600) NULL,
+ notes_type VARCHAR(600) NULL,
+ notes_format VARCHAR(600) NULL,
+ notes_prefix TEXT NULL,
+ notes_prefix_a TEXT NULL,
+ notes_prefix_b TEXT NULL,
+ notes_suffix TEXT NULL,
+ src_filename VARCHAR(256) NOT NULL,
+ src_fingerprint VARCHAR(256) NULL,
+ src_filesize VARCHAR(10) NULL,
+ src_wordcount VARCHAR(10) NULL,
+ pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */
+ pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */
+ pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */
+ src_text TEXT NULL,
+ fulltext TEXT NULL,
+ links TEXT NULL
+ );
+ CREATE TABLE doc_objects (
+ lid INTEGER PRIMARY KEY,
+ metadata_tid INTEGER REFERENCES metadata_and_text,
+ ocn SMALLINT,
+ ocnd VARCHAR(6),
+ ocns VARCHAR(6),
+ clean TEXT NULL,
+ body TEXT NULL,
+ book_idx TEXT NULL,
+ seg VARCHAR(256) NULL,
+ lev_an VARCHAR(1),
+ lev SMALLINT NULL,
+ lev0 SMALLINT,
+ lev1 SMALLINT,
+ lev2 SMALLINT,
+ lev3 SMALLINT,
+ lev4 SMALLINT,
+ lev5 SMALLINT,
+ lev6 SMALLINT,
+ lev7 SMALLINT,
+ en_a SMALLINT NULL,
+ en_z SMALLINT NULL,
+ en_a_asterisk SMALLINT NULL,
+ en_z_asterisk SMALLINT NULL,
+ en_a_plus SMALLINT NULL,
+ en_z_plus SMALLINT NULL,
+ t_of VARCHAR(16),
+ t_is VARCHAR(16),
+ node VARCHAR(16) NULL,
+ parent VARCHAR(16) NULL,
+ digest_clean CHAR(256),
+ digest_all CHAR(256),
+ types CHAR(1) NULL
+ );
+ CREATE INDEX idx_ocn ON doc_objects(ocn);
+ CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);
+ CREATE INDEX idx_digest_all ON doc_objects(digest_all);
+ CREATE INDEX idx_clean ON doc_objects(clean);
+ CREATE INDEX idx_lev0 ON doc_objects(lev0);
+ CREATE INDEX idx_lev1 ON doc_objects(lev1);
+ CREATE INDEX idx_lev2 ON doc_objects(lev2);
+ CREATE INDEX idx_lev3 ON doc_objects(lev3);
+ CREATE INDEX idx_lev4 ON doc_objects(lev4);
+ CREATE INDEX idx_lev5 ON doc_objects(lev5);
+ CREATE INDEX idx_lev6 ON doc_objects(lev6);
+ CREATE INDEX idx_title ON metadata_and_text(title);
+ CREATE INDEX idx_author ON metadata_and_text(creator_author);
+ CREATE INDEX idx_filename ON metadata_and_text(src_filename);
+ CREATE INDEX idx_language ON metadata_and_text(language_document_char);
+ CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);
+ ¶",);
+ return _sql_instruct;
+ }
+}
+template SQLiteInsertMetadata() {
+ string SQLiteInsertMetadata(I)(
+ auto ref I doc_matters,
+ ) {
+ string _insert_metadata;
+ _insert_metadata = format(q"¶
+ INSERT INTO metadata_and_text (
+ src_filename,
+ src_composite_id_per_txt,
+ src_composite_id_per_pod,
+ title,
+ title_main,
+ title_sub,
+ title_short,
+ title_edition,
+ title_language,
+ classify_dewey,
+ classify_keywords,
+ classify_loc,
+ classify_subject,
+ classify_topic_register,
+ creator_author,
+ creator_author_email,
+ creator_illustrator,
+ creator_translator,
+ date_added_to_site,
+ date_available,
+ date_created,
+ date_issued,
+ date_modified,
+ date_published,
+ date_valid,
+ identifier_isbn,
+ identifier_oclc,
+ language_document,
+ language_document_char,
+ notes_abstract,
+ notes_description,
+ original_publisher,
+ original_language,
+ original_language_char,
+ original_source,
+ original_title,
+ publisher,
+ rights_copyright,
+ rights_copyright_audio,
+ rights_copyright_cover,
+ rights_copyright_illustrations,
+ rights_copyright_photographs,
+ rights_copyright_text,
+ rights_copyright_translation,
+ rights_copyright_video,
+ rights_license
+ )
+ VALUES (
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s',
+ '%s'
+ );
+ ¶",
+ SQLinsertDelimiter!()(doc_matters.src.filename),
+ SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc),
+ SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video),
+ SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license)
+ );
+ return _insert_metadata;
+ }
+}
+template SQLiteInsertDocObjectsLoop() {
+ string SQLiteInsertDocObjectsLoop(D,I,X)(
+ auto ref const D doc_abstraction,
+ auto ref I doc_matters,
+ auto ref X _metadata_tid,
+ ) {
+ string insertDocObjectsRow(O)(O obj) {
+ string _insert_doc_objects_row;
+ _insert_doc_objects_row = format(q"¶
+ INSERT INTO doc_objects (
+ metadata_tid,
+ ocn,
+ clean,
+ body,
+ lev,
+ t_of,
+ t_is
+ )
+ VALUES (
+ %d,
+ %s,
+ '%s',
+ '%s',
+ %s,
+ '%s',
+ '%s'
);
- return _insert_metadata;
- }
+ ¶",
+ _metadata_tid,
+ obj.ocn,
+ SQLinsertDelimiter!()(obj_txt["text"]),
+ SQLinsertDelimiter!()(obj_txt["html"]),
+ obj.heading_lev_markup,
+ obj.is_of,
+ obj.is_a,
+ );
+ return _insert_doc_objects_row;
}
- template SQLiteInsertDocObjectsLoop() {
- string SQLiteInsertDocObjectsLoop(P,I)(
- auto ref P doc_parts,
- auto ref I _metadata_tid,
- ) {
- string insertDocObjectsRow(O)(O obj) {
- string _insert_doc_objects_row;
- _insert_doc_objects_row = format(q"¶
- INSERT INTO doc_objects (
- metadata_tid,
- ocn,
- clean,
- body,
- lev,
- t_of,
- t_is
- )
- VALUES (
- %d,
- %s,
- '%s',
- '%s',
- %s,
- '%s',
- '%s'
- );
- ¶",
- _metadata_tid,
- obj.ocn,
- SQLinsertDelimiter!()(obj_txt["text"]),
- SQLinsertDelimiter!()(obj_txt["html"]),
- obj.heading_lev_markup,
- obj.is_of,
- obj.is_a,
- );
- return _insert_doc_objects_row;
- }
- auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);
- string[string] obj_txt;
- string doc_text;
- string[] _insert_doc_objects;
- foreach (part; doc_parts) {
- foreach (obj; doc_abstraction[part]) {
- switch (obj.of_part) {
- case "frontmatter": assert(part == "head", part);
- switch (obj.is_of) {
- case "para":
- switch (obj.is_a) {
- case "heading":
- obj_txt = format_and_sqlite_load.heading(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
- }
- break;
+ auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);
+ string[string] obj_txt;
+ string doc_text;
+ string[] _insert_doc_objects;
+ foreach (part; doc_matters.xml.keys_seq.sql) {
+ foreach (obj; doc_abstraction[part]) {
+ switch (obj.of_part) {
+ case "frontmatter": assert(part == "head", part);
+ switch (obj.is_of) {
+ case "para":
+ switch (obj.is_a) {
+ case "heading":
+ obj_txt = format_and_sqlite_load.heading(obj);
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
}
break;
- case "body": // assert(part == "body", part); // TODO broken
- switch (obj.is_of) {
- case "para":
- switch (obj.is_a) {
- case "heading":
- debug (asserts) { // TODO consider and fix or remove
- if (part != "body") {
- writeln(__LINE__, ": ", obj.text);
- }
- }
- obj_txt = format_and_sqlite_load.heading(obj);
- break;
- case "para":
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- case "block":
- switch (obj.is_a) {
- case "quote":
- obj_txt = format_and_sqlite_load.quote(obj);
- break;
- case "group":
- obj_txt = format_and_sqlite_load.group(obj);
- break;
- case "block":
- obj_txt = format_and_sqlite_load.block(obj);
- break;
- case "poem": // double check on keeping both poem & verse
- break;
- case "verse":
- obj_txt = format_and_sqlite_load.verse(obj);
- break;
- case "code":
- obj_txt = format_and_sqlite_load.code(obj);
- break;
- case "table":
- obj_txt = format_and_sqlite_load.table(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
+ }
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
+ }
+ break;
+ }
+ break;
+ case "body": // assert(part == "body", part); // TODO broken
+ switch (obj.is_of) {
+ case "para":
+ switch (obj.is_a) {
+ case "heading":
+ debug (asserts) { // TODO consider and fix or remove
+ if (part != "body") {
+ writeln(__LINE__, ": ", obj.text);
}
- break;
}
+ obj_txt = format_and_sqlite_load.heading(obj);
break;
- case "backmatter":
- assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part);
- switch (obj.is_of) {
- case "para":
- switch (obj.is_a) {
- case "heading":
- obj_txt = format_and_sqlite_load.heading(obj);
- break;
- case "endnote": assert(part == "endnotes", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "glossary": assert(part == "glossary", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "bibliography": assert(part == "bibliography", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "bookindex": assert(part == "bookindex_seg", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "blurb": assert(part == "blurb", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
- }
- break;
+ case "para":
+ obj_txt = format_and_sqlite_load.para(obj);
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
}
break;
- case "comment":
+ }
+ break;
+ case "block":
+ switch (obj.is_a) {
+ case "quote":
+ obj_txt = format_and_sqlite_load.quote(obj);
+ break;
+ case "group":
+ obj_txt = format_and_sqlite_load.group(obj);
+ break;
+ case "block":
+ obj_txt = format_and_sqlite_load.block(obj);
+ break;
+ case "poem": // double check on keeping both poem & verse
+ break;
+ case "verse":
+ obj_txt = format_and_sqlite_load.verse(obj);
+ break;
+ case "code":
+ obj_txt = format_and_sqlite_load.code(obj);
+ break;
+ case "table":
+ obj_txt = format_and_sqlite_load.table(obj);
break;
default:
if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from
writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from
}
break;
}
- if (obj.is_a == "heading") {
- if ((doc_matters.opt.action.verbose)) {
- writeln(
- "markup: ", obj.heading_lev_markup,
- "> ", obj.dom_markedup,
- "; collapsed: ", obj.heading_lev_collapsed,
- "> ", obj.dom_collapsed,
- "; ocn: ", obj.ocn,
- " node: ", obj.node,
- "; parent: ", obj.parent_lev_markup,
- "; ocn: ", obj.parent_ocn,
- "; ",
- );
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
+ }
+ break;
+ }
+ break;
+ case "backmatter":
+ assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part);
+ switch (obj.is_of) {
+ case "para":
+ switch (obj.is_a) {
+ case "heading":
+ obj_txt = format_and_sqlite_load.heading(obj);
+ break;
+ case "endnote": assert(part == "endnotes", part);
+ obj_txt = format_and_sqlite_load.para(obj);
+ break;
+ case "glossary": assert(part == "glossary", part);
+ obj_txt = format_and_sqlite_load.para(obj);
+ break;
+ case "bibliography": assert(part == "bibliography", part);
+ obj_txt = format_and_sqlite_load.para(obj);
+ break;
+ case "bookindex": assert(part == "bookindex_seg", part);
+ obj_txt = format_and_sqlite_load.para(obj);
+ break;
+ case "blurb": assert(part == "blurb", part);
+ obj_txt = format_and_sqlite_load.para(obj);
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
}
+ break;
}
- if (!(obj.is_a == "comment")) {
- _insert_doc_objects ~= insertDocObjectsRow(obj);
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
}
- } // loop closes
+ break;
+ }
+ break;
+ case "comment":
+ break;
+ default:
+ if ((doc_matters.opt.action.debug_do)) {
+ writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from
+ writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
+ writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from
+ }
+ break;
}
- return _insert_doc_objects.join.to!(char[]).toUTF8;
- }
+ if (obj.is_a == "heading") {
+ if ((doc_matters.opt.action.verbose)) {
+ writeln(
+ "markup: ", obj.heading_lev_markup,
+ "> ", obj.dom_markedup,
+ "; collapsed: ", obj.heading_lev_collapsed,
+ "> ", obj.dom_collapsed,
+ "; ocn: ", obj.ocn,
+ " node: ", obj.node,
+ "; parent: ", obj.parent_lev_markup,
+ "; ocn: ", obj.parent_ocn,
+ "; ",
+ );
+ }
+ }
+ if (!(obj.is_a == "comment")) {
+ _insert_doc_objects ~= insertDocObjectsRow(obj);
+ }
+ } // loop closes
}
- SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);
+ return _insert_doc_objects.join.to!(char[]).toUTF8;
}
}
template SQLiteTablesCreate() {
@@ -1066,7 +1103,8 @@ template SQLiteTablesCreate() {
DROP TABLE IF EXISTS urls;
CREATE TABLE metadata_and_text (
tid INTEGER PRIMARY KEY,
- /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
+ src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
title VARCHAR(800) NOT NULL,
title_main VARCHAR(400) NOT NULL,
title_sub VARCHAR(400) NULL,
diff --git a/src/sdp/output/sqlite_discrete.d b/src/sdp/output/sqlite_discrete.d
deleted file mode 100644
index 40e0b15..0000000
--- a/src/sdp/output/sqlite_discrete.d
+++ /dev/null
@@ -1,1023 +0,0 @@
-module sdp.output.sqlite_discrete;
-import sdp.output;
-import
- std.file,
- std.conv : to;
-template SQLiteDiscreteDbRun() {
- void SQLiteDiscreteDbRun(Db,St,O)(
- auto ref Db db,
- auto ref St db_statement,
- auto ref O opt_action,
- string note,
- ) {
- debug(sql_statement) {
- writeln(db_statement);
- }
- try {
- db.run(
- "\nBEGIN;\n" ~
- db_statement ~
- "\nCOMMIT;\n"
- );
- } catch (ErrnoException ex) {
- writeln("ERROR SQLite : ", ex);
- } catch (Exception ex) {
- writeln("ERROR SQLite : ", ex);
- }
- if (opt_action.debug_do) {
- writeln(note);
- if (opt_action.verbose) {
- writeln(db_statement);
- }
- }
- }
-}
-template SQLiteDiscreteBuildTablesAndPopulate() {
- void SQLiteDiscreteBuildTablesAndPopulate(D,I)(
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
- ) {
- import d2sqlite3;
- import std.typecons : Nullable;
- mixin SiSUoutputRgxInit;
- static auto rgx = Rgx();
- auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language);
- pth_sqlite.base.mkdirRecurse;
- auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename));
- template SQLiteDiscreteDbStatementComposite() {
- void SQLiteDiscreteDbStatementComposite(Db,D,I)(
- auto ref Db db,
- auto ref const D doc_abstraction,
- auto ref I doc_matters,
- ) {
- {
- string _db_statement;
- {
- _db_statement ~= SQLiteDiscreteTablesReCreate!()();
- _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters);
- _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX
- SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects");
- }
- db.close;
- }
- }
- }
- template SQLinsertDelimiter() {
- auto SQLinsertDelimiter(string _txt) {
- _txt = _txt
- .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0");
- return _txt;
- }
- }
- template SQLiteFormatAndLoadObject() {
- auto SQLiteFormatAndLoadObject(I)(
- auto ref I doc_matters,
- ) {
- mixin SiSUoutputRgxInit;
- struct sqlite_format_and_load_objects {
- auto generic_munge_sanitize_text_for_search(
- string _txt,
- ) {
- string _notes;
- string _urls;
- if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
- foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
- _notes ~= "\n" ~ m["text"];
- }
- _txt = _txt.replaceAll(rgx.inline_notes_al_gen, "");
- }
- if (_txt.matchFirst(rgx.inline_link)) {
- foreach (m; _txt.matchAll(rgx.inline_link)) {
- if (m["link"].match(rgx.url)) {
- _urls ~= "\n" ~ m["link"];
- }
- }
- _txt = _txt.replaceAll(rgx.inline_link_clean, "");
- }
- if (_notes.length > 0) {
- _txt ~= _notes;
- }
- if (_urls.length > 0) {
- _txt ~= _urls;
- }
- if (doc_matters.opt.action.debug_do) {
- writeln(_txt, "\n");
- }
- debug(sql_text_clean) {
- writeln(_txt);
- }
- return _txt;
- }
- auto munge_html(O)(
- auto return ref const O obj,
- ) {
- string _html_special_characters(string _txt){
- _txt = _txt
- .replaceAll(rgx.xhtml_ampersand, "&")
- .replaceAll(rgx.xhtml_quotation, """)
- .replaceAll(rgx.xhtml_less_than, "<")
- .replaceAll(rgx.xhtml_greater_than, ">")
- .replaceAll(rgx.nbsp_char, " ")
- .replaceAll(rgx.xhtml_line_break, "<br />");
- return _txt;
- }
- string _html_font_face(string _txt){
- _txt = _txt
- .replaceAll(rgx.inline_emphasis, "<em>$1</em>")
- .replaceAll(rgx.inline_bold, "<b>$1</b>")
- .replaceAll(rgx.inline_underscore, "<u>$1</u>")
- .replaceAll(rgx.inline_italics, "<i>$1</i>")
- .replaceAll(rgx.inline_superscript, "<sup>$1</sup>")
- .replaceAll(rgx.inline_subscript, "<sub>$1</sub>")
- .replaceAll(rgx.inline_strike, "<del>$1</del>")
- .replaceAll(rgx.inline_insert, "<ins>$1</ins>")
- .replaceAll(rgx.inline_mono, "<tt>$1</tt>")
- .replaceAll(rgx.inline_cite, "<cite>$1</cite>");
- return _txt;
- }
- string _notes;
- string _urls;
- string _txt = _html_font_face(_html_special_characters(obj.text));
- if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
- foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
- _notes ~= "\n" ~ m["text"];
- }
- _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "<sup>$1</sup> ");
- }
- if (_txt.matchFirst(rgx.inline_link)) {
- foreach (m; _txt.matchAll(rgx.inline_link)) {
- }
- _txt = _txt.replaceAll(rgx.inline_link_clean, "");
- }
- if (_notes.length > 0) {
- _txt ~= _notes;
- }
- if (doc_matters.opt.action.debug_do) {
- writeln(_txt, "\n");
- }
- return _txt;
- }
- string html_special_characters(string _txt){
- _txt = _txt
- .replaceAll(rgx.xhtml_ampersand, "&")
- .replaceAll(rgx.xhtml_quotation, """)
- .replaceAll(rgx.xhtml_less_than, "<")
- .replaceAll(rgx.xhtml_greater_than, ">")
- .replaceAll(rgx.nbsp_char, " ")
- .replaceAll(rgx.xhtml_line_break, "<br />");
- return _txt;
- }
- string html_special_characters_code(string _txt){
- _txt = _txt
- .replaceAll(rgx.xhtml_ampersand, "&")
- .replaceAll(rgx.xhtml_quotation, """)
- .replaceAll(rgx.xhtml_less_than, "<")
- .replaceAll(rgx.xhtml_greater_than, ">")
- .replaceAll(rgx.nbsp_char, " ");
- return _txt;
- }
- string html_font_face(string _txt){
- _txt = _txt
- .replaceAll(rgx.inline_emphasis, "<em>$1</em>")
- .replaceAll(rgx.inline_bold, "<b>$1</b>")
- .replaceAll(rgx.inline_underscore, "<u>$1</u>")
- .replaceAll(rgx.inline_italics, "<i>$1</i>")
- .replaceAll(rgx.inline_superscript, "<sup>$1</sup>")
- .replaceAll(rgx.inline_subscript, "<sub>$1</sub>")
- .replaceAll(rgx.inline_strike, "<del>$1</del>")
- .replaceAll(rgx.inline_insert, "<ins>$1</ins>")
- .replaceAll(rgx.inline_mono, "<tt>$1</tt>")
- .replaceAll(rgx.inline_cite, "<cite>$1</cite>");
- return _txt;
- }
- auto html_heading(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s"><b>
- %s
- </b></p>¶",
- obj.is_a,
- _txt,
- );
- return o;
- }
- auto html_para(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt;
- string o = format(q"¶<p class="%s" indent="h%si%s">
- %s
- </p>¶",
- obj.is_a,
- obj.indent_hang,
- obj.indent_base,
- _txt
- );
- return o;
- }
- auto html_quote(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s">
- %s
- </p>¶",
- obj.is_a,
- _txt
- );
- return o;
- }
- auto html_group(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s">
- %s
- </p>¶",
- obj.is_a,
- _txt
- );
- return o;
- }
- auto html_block(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶
- <p class="%s">%s</p>¶",
- obj.is_a,
- _txt.stripRight
- );
- return o;
- }
- auto html_verse(O)(
- auto return ref const O obj,
- ) {
- string _txt = munge_html(obj);
- string o = format(q"¶<p class="%s">%s</p>¶",
- obj.is_a,
- _txt
- );
- return o;
- }
- auto html_code(O)(
- auto return ref const O obj,
- ) {
- string _txt = html_special_characters_code(obj.text);
- string o = format(q"¶<p class="%s">%s</p>¶",
- obj.is_a,
- _txt
- );
- return o;
- }
- auto html_table(O)(
- auto return ref const O obj,
- ) {
- auto _tablarize(O)(
- auto return ref const O obj,
- string _txt,
- ) {
- string[] _table_rows = _txt.split(rgx.table_delimiter_row);
- string[] _table_cols;
- string _table;
- string _tablenote;
- foreach(row_idx, row; _table_rows) {
- _table_cols = row.split(rgx.table_delimiter_col);
- _table ~= "<tr>";
- foreach(col_idx, cell; _table_cols) {
- if ((_table_cols.length == 1)
- && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx)
- _tablenote ~= cell;
- } else {
- string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td";
- string _align = ("style=\"text-align:"
- ~ ((obj.table_column_aligns[col_idx] == "l")
- ? "left\"" : "right\""));
- _table ~= "<"
- ~ _col_is
- ~ " width=\""
- ~ obj.table_column_widths[col_idx].to!string
- ~ "%\" "
- ~ _align
- ~ ">";
- _table ~= cell;
- _table ~= "</"
- ~ _col_is
- ~ ">";
- }
- }
- _table ~= "</tr>";
- }
- auto t = tuple(
- _table,
- _tablenote,
- );
- return t;
- }
- string _txt = munge_html(obj);
- auto t = _tablarize(obj, _txt);
- _txt = t[0];
- string _note = t[1];
- string o = format(q"¶<p class="%s">
- <table summary="normal text css" width="95%%" border="0" bgcolor="white" cellpadding="2" align="center">
- %s
- </table>
- %s
- </p>¶",
- obj.is_a,
- _txt,
- _note
- );
- return o;
- }
- auto sqlite_load_string(O,Dm)(
- auto return ref const O obj,
- auto return ref Dm doc_matters,
- ) {
- string o;
- return o;
- }
- auto postgresql_load_string(O,Dm)(
- auto return ref const O obj,
- auto return ref Dm doc_matters,
- ) {
- string o;
- return o;
- }
- string sqlite_statement(O)(
- auto return ref const O obj,
- string _txt,
- string _html,
- ) {
- void _sql_exe(O)(
- string _sql,
- ) {
- writeln(_html);
- writeln(_sql);
- }
- string _sql;
- return _sql;
- }
- auto heading(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_heading(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto para(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_para(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto quote(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_quote(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto group(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_group(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto block(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_block(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto verse(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_verse(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto code(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_code(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- auto table(O)(
- auto return ref const O obj,
- ) {
- string[string] obj_txt = [
- "text": generic_munge_sanitize_text_for_search(obj.text),
- "html": html_table(obj)
- ];
- if (doc_matters.opt.action.debug_do) {
- debug(sql_txt) {
- writeln(obj_txt["text"]);
- }
- debug(sql_html) {
- writeln(obj_txt["html"]);
- }
- } else {
- // load sql
- }
- return obj_txt;
- }
- }
- return sqlite_format_and_load_objects();
- }
- }
- template SQLiteDiscreteTablesReCreate() {
- string SQLiteDiscreteTablesReCreate()() {
- string _sql_instruct;
- _sql_instruct = format(q"¶
- DROP INDEX IF EXISTS ocn;
- DROP INDEX IF EXISTS digest_clean;
- DROP INDEX IF EXISTS digest_all;
- DROP INDEX IF EXISTS clean;
- DROP INDEX IF EXISTS lev0;
- DROP INDEX IF EXISTS lev1;
- DROP INDEX IF EXISTS lev2;
- DROP INDEX IF EXISTS lev3;
- DROP INDEX IF EXISTS lev4;
- DROP INDEX IF EXISTS lev5;
- DROP INDEX IF EXISTS lev6;
- DROP INDEX IF EXISTS title;
- DROP INDEX IF EXISTS creator_author;
- DROP INDEX IF EXISTS src_filename;
- DROP INDEX IF EXISTS language_document_char;
- DROP INDEX IF EXISTS classify_topic_register;
- DROP TABLE IF EXISTS metadata_and_text;
- DROP TABLE IF EXISTS doc_objects;
- DROP TABLE IF EXISTS urls;
- CREATE TABLE metadata_and_text (
- tid INTEGER PRIMARY KEY,
- /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */
- title VARCHAR(800) NOT NULL,
- title_main VARCHAR(400) NOT NULL,
- title_sub VARCHAR(400) NULL,
- title_short VARCHAR(400) NULL,
- title_edition VARCHAR(10) NULL,
- title_note VARCHAR(2500) NULL,
- title_language VARCHAR(100) NULL,
- title_language_char VARCHAR(6) NULL,
- creator_author VARCHAR(600) NOT NULL,
- creator_author_email VARCHAR(100) NULL,
- creator_author_hon VARCHAR(100) NULL,
- creator_author_nationality VARCHAR(100) NULL,
- creator_editor VARCHAR(600) NULL,
- creator_contributor VARCHAR(600) NULL,
- creator_illustrator VARCHAR(600) NULL,
- creator_photographer VARCHAR(600) NULL,
- creator_translator VARCHAR(600) NULL,
- creator_prepared_by VARCHAR(600) NULL,
- creator_digitized_by VARCHAR(600) NULL,
- creator_audio VARCHAR(600) NULL,
- creator_video VARCHAR(600) NULL,
- language_document VARCHAR(100) NULL,
- language_document_char VARCHAR(6) NOT NULL,
- language_original VARCHAR(100) NULL,
- language_original_char VARCHAR(6) NULL,
- date_added_to_site VARCHAR(10) NULL,
- date_available VARCHAR(10) NULL,
- date_created VARCHAR(10) NULL,
- date_issued VARCHAR(10) NULL,
- date_modified VARCHAR(10) NULL,
- date_published VARCHAR(10) NULL,
- date_valid VARCHAR(10) NULL,
- date_translated VARCHAR(10) NULL,
- date_original_publication VARCHAR(10) NULL,
- date_generated VARCHAR(10) NULL,
- publisher VARCHAR(600) NULL,
- original_publisher VARCHAR(600) NULL,
- original_language VARCHAR(100) NULL,
- original_language_char VARCHAR(6) NULL,
- original_source VARCHAR(600) NULL,
- original_institution VARCHAR(600) NULL,
- original_nationality VARCHAR(100) NULL,
- original_title VARCHAR(800) NULL,
- rights_copyright VARCHAR(2500) NULL,
- rights_copyright_audio VARCHAR(2500) NULL,
- rights_copyright_cover VARCHAR(2500) NULL,
- rights_copyright_illustrations VARCHAR(2500) NULL,
- rights_copyright_photographs VARCHAR(2500) NULL,
- rights_copyright_text VARCHAR(2500) NULL,
- rights_copyright_translation VARCHAR(2500) NULL,
- rights_copyright_video VARCHAR(2500) NULL,
- rights_license VARCHAR(2500) NULL,
- identifier_oclc VARCHAR(30) NULL,
- identifier_isbn VARCHAR(16) NULL,
- classify_topic_register VARCHAR(2500) NULL,
- classify_subject VARCHAR(600) NULL,
- classify_loc VARCHAR(30) NULL,
- classify_dewey VARCHAR(30) NULL,
- classify_keywords VARCHAR(600) NULL,
- notes_abstract TEXT NULL,
- notes_description TEXT NULL,
- notes_comment TEXT NULL,
- notes_coverage VARCHAR(200) NULL,
- notes_relation VARCHAR(200) NULL,
- notes_history VARCHAR(600) NULL,
- notes_type VARCHAR(600) NULL,
- notes_format VARCHAR(600) NULL,
- notes_prefix TEXT NULL,
- notes_prefix_a TEXT NULL,
- notes_prefix_b TEXT NULL,
- notes_suffix TEXT NULL,
- src_filename VARCHAR(256) NOT NULL,
- src_fingerprint VARCHAR(256) NULL,
- src_filesize VARCHAR(10) NULL,
- src_wordcount VARCHAR(10) NULL,
- pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */
- pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */
- pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */
- src_text TEXT NULL,
- fulltext TEXT NULL,
- links TEXT NULL
- );
- CREATE TABLE doc_objects (
- lid INTEGER PRIMARY KEY,
- metadata_tid INTEGER REFERENCES metadata_and_text,
- ocn SMALLINT,
- ocnd VARCHAR(6),
- ocns VARCHAR(6),
- clean TEXT NULL,
- body TEXT NULL,
- book_idx TEXT NULL,
- seg VARCHAR(256) NULL,
- lev_an VARCHAR(1),
- lev SMALLINT NULL,
- lev0 SMALLINT,
- lev1 SMALLINT,
- lev2 SMALLINT,
- lev3 SMALLINT,
- lev4 SMALLINT,
- lev5 SMALLINT,
- lev6 SMALLINT,
- lev7 SMALLINT,
- en_a SMALLINT NULL,
- en_z SMALLINT NULL,
- en_a_asterisk SMALLINT NULL,
- en_z_asterisk SMALLINT NULL,
- en_a_plus SMALLINT NULL,
- en_z_plus SMALLINT NULL,
- t_of VARCHAR(16),
- t_is VARCHAR(16),
- node VARCHAR(16) NULL,
- parent VARCHAR(16) NULL,
- digest_clean CHAR(256),
- digest_all CHAR(256),
- types CHAR(1) NULL
- );
- CREATE INDEX idx_ocn ON doc_objects(ocn);
- CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);
- CREATE INDEX idx_digest_all ON doc_objects(digest_all);
- CREATE INDEX idx_clean ON doc_objects(clean);
- CREATE INDEX idx_lev0 ON doc_objects(lev0);
- CREATE INDEX idx_lev1 ON doc_objects(lev1);
- CREATE INDEX idx_lev2 ON doc_objects(lev2);
- CREATE INDEX idx_lev3 ON doc_objects(lev3);
- CREATE INDEX idx_lev4 ON doc_objects(lev4);
- CREATE INDEX idx_lev5 ON doc_objects(lev5);
- CREATE INDEX idx_lev6 ON doc_objects(lev6);
- CREATE INDEX idx_title ON metadata_and_text(title);
- CREATE INDEX idx_author ON metadata_and_text(creator_author);
- CREATE INDEX idx_filename ON metadata_and_text(src_filename);
- CREATE INDEX idx_language ON metadata_and_text(language_document_char);
- CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register);
- ¶",);
- return _sql_instruct;
- }
- }
- template SQLiteDiscreteInsertMetadata() {
- string SQLiteDiscreteInsertMetadata(I)(
- auto ref I doc_matters,
- ) {
- string _insert_metadata;
- _insert_metadata = format(q"¶
- INSERT INTO metadata_and_text (
- src_filename,
- title,
- title_main,
- title_sub,
- title_short,
- title_edition,
- title_language,
- classify_dewey,
- classify_keywords,
- classify_loc,
- classify_subject,
- classify_topic_register,
- creator_author,
- creator_author_email,
- creator_illustrator,
- creator_translator,
- date_added_to_site,
- date_available,
- date_created,
- date_issued,
- date_modified,
- date_published,
- date_valid,
- identifier_isbn,
- identifier_oclc,
- language_document,
- language_document_char,
- notes_abstract,
- notes_description,
- original_publisher,
- original_language,
- original_language_char,
- original_source,
- original_title,
- publisher,
- rights_copyright,
- rights_copyright_audio,
- rights_copyright_cover,
- rights_copyright_illustrations,
- rights_copyright_photographs,
- rights_copyright_text,
- rights_copyright_translation,
- rights_copyright_video,
- rights_license
- )
- VALUES (
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s',
- '%s'
- );
- ¶",
- SQLinsertDelimiter!()(doc_matters.src.filename),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video),
- SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license)
- );
- return _insert_metadata;
- }
- }
- template SQLiteDiscreteInsertDocObjectsLoop() {
- string SQLiteDiscreteInsertDocObjectsLoop(P,I)(
- auto ref P doc_parts,
- auto ref I tid,
- ) {
- string insertDocObjectsRow(O)(O obj) {
- string _insert_doc_objects_row;
- _insert_doc_objects_row = format(q"¶
- INSERT INTO doc_objects (
- metadata_tid,
- ocn,
- clean,
- body,
- lev,
- t_of,
- t_is
- )
- VALUES (
- %d,
- %s,
- '%s',
- '%s',
- %s,
- '%s',
- '%s'
- );
- ¶",
- 1,
- obj.ocn,
- SQLinsertDelimiter!()(obj_txt["text"]),
- SQLinsertDelimiter!()(obj_txt["html"]),
- obj.heading_lev_markup,
- obj.is_of,
- obj.is_a,
- );
- return _insert_doc_objects_row;
- }
- auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);
- string[string] obj_txt;
- string doc_text;
- string[] _insert_doc_objects;
- foreach (part; doc_parts) {
- foreach (obj; doc_abstraction[part]) {
- switch (obj.of_part) {
- case "frontmatter": assert(part == "head", part);
- switch (obj.is_of) {
- case "para":
- switch (obj.is_a) {
- case "heading":
- obj_txt = format_and_sqlite_load.heading(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
- }
- break;
- }
- break;
- case "body": // assert(part == "body", part); // TODO broken
- switch (obj.is_of) {
- case "para":
- switch (obj.is_a) {
- case "heading":
- debug (asserts) { // TODO consider and fix or remove
- if (part != "body") {
- writeln(__LINE__, ": ", obj.text);
- }
- }
- obj_txt = format_and_sqlite_load.heading(obj);
- break;
- case "para":
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- case "block":
- switch (obj.is_a) {
- case "quote":
- obj_txt = format_and_sqlite_load.quote(obj);
- break;
- case "group":
- obj_txt = format_and_sqlite_load.group(obj);
- break;
- case "block":
- obj_txt = format_and_sqlite_load.block(obj);
- break;
- case "poem": // double check on keeping both poem & verse
- break;
- case "verse":
- obj_txt = format_and_sqlite_load.verse(obj);
- break;
- case "code":
- obj_txt = format_and_sqlite_load.code(obj);
- break;
- case "table":
- obj_txt = format_and_sqlite_load.table(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
- }
- break;
- }
- break;
- case "backmatter":
- assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part);
- switch (obj.is_of) {
- case "para":
- switch (obj.is_a) {
- case "heading":
- obj_txt = format_and_sqlite_load.heading(obj);
- break;
- case "endnote": assert(part == "endnotes", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "glossary": assert(part == "glossary", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "bibliography": assert(part == "bibliography", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "bookindex": assert(part == "bookindex_seg", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- case "blurb": assert(part == "blurb", part);
- obj_txt = format_and_sqlite_load.para(obj);
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- }
- break;
- }
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_of);
- }
- break;
- }
- break;
- case "comment":
- break;
- default:
- if ((doc_matters.opt.action.debug_do)) {
- writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from
- writeln(__FILE__, ":", __LINE__, ": ", obj.is_a);
- writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from
- }
- break;
- }
- if (obj.is_a == "heading") {
- if ((doc_matters.opt.action.verbose)) {
- writeln(
- "markup: ", obj.heading_lev_markup,
- "> ", obj.dom_markedup,
- "; collapsed: ", obj.heading_lev_collapsed,
- "> ", obj.dom_collapsed,
- "; ocn: ", obj.ocn,
- " node: ", obj.node,
- "; parent: ", obj.parent_lev_markup,
- "; ocn: ", obj.parent_ocn,
- "; ",
- );
- }
- }
- if (!(obj.is_a == "comment")) {
- _insert_doc_objects ~= insertDocObjectsRow(obj);
- }
- } // loop closes
- }
- return _insert_doc_objects.join.to!(char[]).toUTF8;
- }
- }
- SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);
- }
-}
--
cgit v1.2.3