From 960c3088bc88f2db879154053280b06c160d4d70 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Tue, 21 Apr 2015 14:45:52 -0400 Subject: lib/sisu/*, single libs directory (c&d gone) (7) * removed lib/sisu/{current,develop} dir branches v7 (v5 & v6 retired) * simplify dir structure, offer single version per snapshot * have enjoyed carrying stable and development versions v5 & v6 in a single tarball, may return to this structure --- lib/sisu/xhtml_epub2_concordance.rb | 322 ++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 lib/sisu/xhtml_epub2_concordance.rb (limited to 'lib/sisu/xhtml_epub2_concordance.rb') diff --git a/lib/sisu/xhtml_epub2_concordance.rb b/lib/sisu/xhtml_epub2_concordance.rb new file mode 100644 index 00000000..ab50a7f8 --- /dev/null +++ b/lib/sisu/xhtml_epub2_concordance.rb @@ -0,0 +1,322 @@ +# encoding: utf-8 +=begin + +* Name: SiSU + +** Description: documents, structuring, processing, publishing, search +*** epub concordance file (wordmap, linked index of words in document) + +** Author: Ralph Amissah + + + +** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah, + All Rights Reserved. + +** License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + +** SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + +** Hompages: + + + +** Git + + + +=end +module SiSU_XHTML_EPUB2_Concordance + require_relative 'se_hub_particulars' # se_hub_particulars.rb + include SiSU_Particulars + require_relative 'se' # se.rb + include SiSU_Env + require_relative 'xhtml_parts' # xhtml_parts.rb + require_relative 'xhtml_epub2_format' # xhtml_epub2_format.rb + include SiSU_XHTML_EPUB2_Format + class Source + def initialize(opt) + @opt=opt + @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt) + end + def read + begin + @env,@md=@particulars.env,@particulars.md + wordmax=@env.concord_max + unless @md.wc_words.nil? + if @md.wc_words < wordmax + SiSU_XHTML_EPUB2_Concordance::Source::Words.new(@particulars).songsheet + else + SiSU_Screen::Ansi.new( + @md.opt.act[:color_state][:set], + "*WARN* concordance skipped, large document has over #{wordmax} words (#{@md.wc_words})" + ).warn unless @md.opt.act[:quiet][:set]==:on + end + else + SiSU_Screen::Ansi.new( + @md.opt.act[:color_state][:set], + "*WARN* wc (word count) is off, concordance will be processed for all files including those over the max set size of: #{wordmax} words" + ).warn unless @md.opt.act[:quiet][:set]==:on + SiSU_XHTML_EPUB2_Concordance::Source::Words.new(@particulars).songsheet + end + rescue + SiSU_Errors::Rescued.new($!,$@,@md.opt.selections.str,@md.fns).location do + __LINE__.to_s + ':' + __FILE__ + end + ensure + end + end + private + class DocTitle + #revisit, both requires (html & xml_shared) needed for stand alone operation (sisu -w [filename]) + require_relative 'xhtml_epub2' # xhtml_epub2.rb + def initialize(particulars) + @particulars,@md=particulars,particulars.md + @data=SiSU_XHTML_EPUB2::Source::XHTML_Environment.new(particulars).tuned_file_instructions + @fnb=@md.fnb + @lex_button=%{SiSU home} + @doc_details =<#{$ep[:hsp]}

#{@md.title.full}

#{@md.creator.author}

+WOK + end + def create + @css=SiSU_Env::CSS_Stylesheet.new(@particulars.md) + format_head_toc=SiSU_XHTML_EPUB2_Format::HeadToc.new(@md) + dochead=format_head_toc.head + < + #{@doc_details} +

Word index links are to html versions of the text the segmented version followed by the scroll (single document) version.
[For segmented text references [T1], [T2] or [T3] appearing without a link, indicates that the word appears in a title (or subtitle) of the text (that is identifiable by the appended object citation number).]

+

(The word listing/index is Case sensitive: Capitalized words appear before lower case)

+

+ word (number of occurences)
linked references to word within document
+ [if number of occurences exceed number of references - word occurs more than once in at least one reference. Footnote/endnotes are either assigned to the paragraph from which they are referenced or ignored, so it is relevant to check the footnotes referenced from within a paragraph as well.] +

+

+ (After the page is fully loaded) you can jump directly to a word by appending a hash (#) and the word to the url for this text, (do not forget that words are case sensitive, and may be listed twice (starting with and without an upper case letter)), #your_word # [#{$ep[:hsp]}http://[web host]/#{@fnb}/concordance.html#your_word#{$ep[:hsp]}] +

+WOK + end + end + class Word + @@word_previous='' + def initialize(word,freq) + @word,@freq=word,freq + end + def html + w=if @word.capitalize==@@word_previous + %{\n

#{@word}

(#{@freq})

\n\t

} + else n=@word.strip.gsub(/\s+/,'_') #also need to convert extended character set to html + %{\n

#{@word}

(#{@freq})

\n\t

} + end + @@word_previous=@word.capitalize + w + end + end + class Words + require_relative 'xhtml_epub2_format' # xhtml_epub2_format.rb + include SiSU_XHTML_EPUB2_Format + require_relative 'se' # se.rb + include SiSU_Screen + def initialize(particulars) + @particulars=particulars + begin + @env,@md,@ao_array=particulars.env,particulars.md,particulars.ao_array + @path="#{@env.processing_path.epub}" + @freq=Hash.new(0) + @rxp_lv0=/^#{Mx[:lv_o]}0:/ + @rxp_lv1=/^#{Mx[:lv_o]}1:/ + @rxp_lv2=/^#{Mx[:lv_o]}2:/ + @rxp_lv3=/^#{Mx[:lv_o]}3:/ + @rxp_seg=/^#{Mx[:lv_o]}4:(\S+?)#{Mx[:lv_c]}/ + @rxp_title=Regexp.new("^#{Mx[:meta_o]}title#{Mx[:meta_c]}\s*(.+?)\s*$") + @rxp_t0=Regexp.new('^T0') + @rxp_t1=Regexp.new('^T1') + @rxp_t2=Regexp.new('^T2') + @rxp_t3=Regexp.new('^T3') + @rxp_excluded1=/(?:https?|file|ftp):\/\/\S+/ + @rxp_excluded0=/^(?:#{Mx[:fa_bold_o]}|#{Mx[:fa_italics_o]})?(?:to\d+|\d+| |#{Mx[:br_endnotes]}|EOF|#{Mx[:br_eof]}|thumb_\S+|snap_\S+|_+|-+|[(]?(?:ii+|iv|vi+|ix|xi+|xiv|xv|xvi+|xix|xx)[).]?|\S+?_\S+|[\d_]+\w\S+|[\w\d]{1,2}|\d{1,3}\w?|[0-9a-f]{16,64}|\d{2,3}x\d{2,3}|\S{0,2}sha\d|\S{0,3}\d{4}w\d\d|\b\w\d+|\d_all\b|e\.?g\.?)(?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})?$/mi #this regex causes and cures a stack dump in ruby 1.9 !!! + @rgx_splitlist=%r{[—.,;:-]+|#{Mx[:nbsp]}+}mi + @rgx_scanlist=%r{#{Mx[:fa_italics_o]}[a-zA-Z0-9"\s]{2,12}#{Mx[:fa_italics_c]}|#{Mx[:fa_bold_o]}[a-zA-Z0-9"\s]{2,12}#{Mx[:fa_bold_c]}|#{Mx[:url_o]}https?://\S+?#{Mx[:url_c]}|file://\S+|<\S+?>|\w+|[a-zA-Z]+}mi + rescue + SiSU_Errors::Rescued.new($!,$@,@md.opt.selections.str,@md.fns).location do + __LINE__.to_s + ':' + __FILE__ + end + end + end + def songsheet + begin + #fix to use + p __LINE__.to_s + ':' + __FILE__ + p "#{@path}/content/#{@md.fn[:epub_concord]}" + p "#{@md.file.output_path.epub.dir}/#{@md.file.base_filename.epub}" + @file_concordance=File.open("#{@path}/content/#{@md.fn[:epub_concord]}",'w') + map_para + rescue + SiSU_Errors::Rescued.new($!,$@,@md.opt.selections.str,@md.fns).location do + __LINE__.to_s + ':' + __FILE__ + end + ensure + @file_concordance.close + end + end + protected + def location_scroll(wordlocation,show) + @wordlocation=wordlocation + %{#{@wordlocation}; } + end + def location_seg(wordlocation,show) + @wordlocation,@show=wordlocation,show + @word_location_seg=wordlocation.gsub(/(.+?)\#(\d+)/,"\\1#{Sfx[:epub_xhtml]}#o\\2") unless wordlocation.nil? + case @wordlocation + when @rxp_t1 + %{[H]#{@show}, } + when @rxp_t2 + %{[H]#{@show}, } + when @rxp_t3 + %{[H]#{@show}, } + else %{#{@show}, } + end + end + def map_para + @seg,toy=nil,nil + @word_map={} + @ao_array.each do |line| + if defined? line.ocn + if (line.is ==:heading \ + || line.is ==:heading_insert) \ + && line.ln==4 + @seg=line.name + end + if line.ocn.to_s =~/\d+/ then toy=line.ocn.to_s + end + if toy =~/\d+/ \ + and toy !~/^0$/ + line.obj=line.obj.split(@rgx_splitlist).join(' ') #%take in word or other match + for word in line.obj.scan(@rgx_scanlist) #%take in word or other match + word=word.gsub(/#{Mx[:lnk_o]}|#{Mx[:lnk_c]}|#{Mx[:url_o]}|#{Mx[:url_c]}/,''). + gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,''). + gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,''). + gsub(/#{Mx[:gl_o]}#[a-z]+#{Mx[:gl_c]}/,''). + gsub(/#{Mx[:gl_o]}#[0-9]+#{Mx[:gl_c]}/,''). + gsub(/^\S$/,'') + word=nil if word.empty? + word=nil if word =~@rxp_excluded0 #watch + word=nil if word =~@rxp_excluded1 #watch + word=nil if word =~/^\S$/ + if word + word=word.gsub(/#{Mx[:br_nl]}|#{Mx[:br_line]}/,' '). + gsub(/#{Mx[:fa_o]}[a-z]{1,7}#{Mx[:fa_o_c]}|#{Mx[:fa_c_o]}[a-z]{1,7}#{Mx[:fa_c]}/,''). + gsub(/#{Mx[:mk_o]}(?:[0-9a-f]{32}:[0-9a-f]{32}|[0-9a-f]{64}:[0-9a-f]{64})#{Mx[:mk_c]}/,''). + gsub(/#{Mx[:mk_o]}(?:[0-9a-f]{32}|[0-9a-f]{64})#{Mx[:mk_c]}/,''). + gsub(/#{Mx[:en_a_o]}(?:\d|[*+])*|#{Mx[:en_b_o]}(?:\d|[*+])*|#{Mx[:en_a_c]}|#{Mx[:en_b_c]}/mi,''). + gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,'').gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,''). + gsub(/<\/?\S+?>/,''). + gsub(/^\@+/,''). + strip. + gsub(/#{Mx[:tc_p]}.+/,''). + gsub(/[\.,;:"]$/,''). + gsub(/["]/,''). + gsub(/^\s*[\(]/,''). + gsub(/[\(]\s*$/,''). + gsub(/^(?:See|e\.?g\.?).+/,''). + gsub(/^\s*[.,;:]\s*/,''). + strip. + gsub(/^\(?[a-zA-Z]\)$/,''). + gsub(/^\d+(st|nd|rd|th)$/,''). + gsub(/^(\d+\.?)+$/, ''). + gsub(/#{Mx[:mk_o]}|#{Mx[:mk_c]}/,''). + gsub(/:name#\S+/,''). + gsub(/^\S$/,'') + word=nil if word =~/^\S$/ + word=nil if word =~/^\s*$/ #watch + if word + unless word =~/[A-Z][A-Z]/ \ + or word =~/\w+\s\w+/ + word=word.capitalize + end + @freq[word] +=1 + @word_map[word] ||= [] + if line !~@rxp_lv0 \ + and line !~@rxp_lv1 \ + and line !~@rxp_lv2 \ + and line !~@rxp_lv3 + @word_map[word] << location_seg("#{@seg}\##{toy}",toy) + else + @word_map[word] << case line + when @rxp_lv0 then location_seg('T0',toy) + when @rxp_lv1 then location_seg('T1',toy) + when @rxp_lv2 then location_seg('T2',toy) + when @rxp_lv3 then location_seg('T3',toy) + end + end + end + end + end + end + end + end + seg='' + @file_concordance << SiSU_XHTML_EPUB2_Concordance::Source::DocTitle.new(@particulars).create + alph=%W[A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + @file_concordance << '

' + alph.each {|x| @file_concordance << %{#{x},#{$ep[:hsp]}}} + @file_concordance << '

' + letter=alph.shift + @file_concordance << %{\n

A

} + for word in @freq.keys.sort! {|a,b| a.downcase<=>b.downcase} + f=/^(\S)/.match(word)[1] + if letter < f.upcase + while letter < f.upcase + if alph.length > 0 + letter=alph.shift + @file_concordance << %{\n

#{letter}

} + else break + end + end + end + keyword=SiSU_XHTML_EPUB2_Concordance::Source::Word.new(word,@freq[word]).html + if keyword !~ @rxp_excluded0 + if @word_map[word][0] =~ /\d+/ + @file_concordance << %{#{keyword}#{seg}#{@word_map[word].uniq.compact.join}} + end + @file_concordance << '

' + end + # special cases endnotes and header levels 1 - 3 + end + credits=SiSU_Proj_XHTML::Bits.new.credits_sisu_epub + @file_concordance << %{>#{credits}\n} # footer + end + end + end +end +__END__ -- cgit v1.2.3