use \write-toc-csv in a markup to generate a toc.csv

* compatibilty to Liederbuchgenerator
* apply standalone output directly to top level not wrapped in a book

all_base_includes.ly

@@ -1,7 +1,13 @@
 #(define noStandaloneOutput (if (defined? 'noStandaloneOutput) noStandaloneOutput #f))
 
-#(if (defined? 'LAYOUT) #f (load "json_parser.scm"))
+#(if (defined? 'LAYOUT) #f
-#(use-modules (json parser))
+  (let ((scmdir (string-append (dirname (current-filename)) file-name-separator-string "scm" file-name-separator-string)))
+    (load (string-append scmdir "json_parser.scm"))
+    (load (string-append scmdir "resolve_inherits.scm"))
+    (load (string-append scmdir "yaml_parser.scm"))))
+#(define AUTHOR_DATA (if (defined? 'AUTHOR_DATA) AUTHOR_DATA (parse-yml-file "../../lilypond-song-includes/data/authors.yml")))
+#(define SONG_DATA (if (defined? 'SONG_DATA) SONG_DATA (parse-yml-file "../../lilypond-song-includes/data/songs.yml")))
+
 \include "basic_format_and_style_settings.ly"
 \include "eps_file_from_song_dir.ly"
 \include "title_with_category_images.ly"

appendix.ly

@@ -0,0 +1,44 @@
+appendix =
+#(define-void-function (parser location title) (markup?)
+  (define (appendix-item->markup layout props appendix-item)
+    (interpret-markup layout props
+      (markup
+          #:override (cons 'appendixItem:heading (assoc-ref appendix-item "heading"))
+          #:override (cons 'appendixItem:text (assoc-ref appendix-item "text"))
+          (ly:output-def-lookup layout 'appendixItemMarkup))))
+  (ly:book-add-bookpart! (ly:parser-lookup '$current-book)
+  #{
+    \bookpart {
+      \markup { #title }
+      #(for-each
+          (lambda (item)
+            (add-score (ly:make-page-label-marker (string->symbol (car item))))
+            (add-text
+              (make-on-the-fly-markup
+                (lambda (layout props arg) (appendix-item->markup layout props (cdr item)))
+                (make-null-markup)))
+          )
+        (reverse APPENDIX_DATA))
+    }
+  #}))
+
+#(define-markup-command (appendix-ref layout props label) (symbol?)
+  "call page-ref to appendix-item"
+  (interpret-markup layout props
+    (markup #:with-link label
+      #:override (cons 'appendixPage (make-page-ref-markup label "XXX" "?"))
+      (ly:output-def-lookup layout 'appendixReferenceMarkup))))
+
+\paper {
+  appendixItemMarkup = \markup {
+    \left-column {
+      \line { \large \bold \fromproperty #'appendixItem:heading }
+      \vspace #0.2
+      \sans \wordwrap-field #'appendixItem:text
+      \vspace #0.7
+    }
+  }
+  appendixReferenceMarkup = \markup {
+    \fromproperty #'appendixPage
+  }
+}

basic_format_and_style_settings.ly

@@ -105,13 +105,21 @@ romanStanza =
   #(define-music-function (parser location) ()
   #{ \override StanzaNumber.style = #'roman #})
 
+override-stanza =
+#(define-music-function (parser location stanzanumber) (number?)
+#{
+  \once \override StanzaNumber.forced-spacing = #stanzanumber % misuse property "forced-spacing" to override the stanzanumber
+#}
+)
+
 stanza =
 #(define-music-function (parser location stanzanumber) (number?)
 #{
   \once \override StanzaNumber.layer = 23 % set this to signal that there is a real stanza and no repeat signs
   \applyContext
   #(lambda (context)
-    (let* ((stanzastyle (ly:assoc-get 'style (ly:context-grob-definition context 'StanzaNumber)))
+    (let* ((stanzanumber (ly:assoc-get 'forced-spacing (ly:context-grob-definition context 'StanzaNumber) stanzanumber))
+           (stanzastyle (ly:assoc-get 'style (ly:context-grob-definition context 'StanzaNumber)))
            (formattedStanzaNumber (format #f (if (eq? stanzastyle 'roman) romanStanzaFormat stanzaFormat) stanzanumber)))
         (ly:context-set-property! context 'stanza formattedStanzaNumber)))
 #}
@@ -147,4 +155,32 @@ repStopWithTag = \lyricmode {
     \once \override StanzaNumber.direction = 1
     \set stanza = \markup { \repStop }
   }
-}
+}
+
+rightHyphen = \lyricmode {
+  \once \override StanzaNumber.font-series = #'normal
+  \once \override StanzaNumber.direction = 1
+  \set stanza = "-"
+}
+
+leftHyphen = \lyricmode {
+  \once \override StanzaNumber.font-series = #'normal
+  \set stanza = "-"
+}
+
+multiVerseSkips =
+#(define-music-function (parser location skips) (number?)
+  #{ \tag #'multiVerse { \repeat unfold #skips { \skip4 } } #})
+
+alt =
+#(define-music-function (parser location a b) (ly:music? ly:music?)
+  #{ \tag #'firstVerse { #a } \tag #'multiVerse { #b } #})
+
+updown =
+#(define-music-function (parser location word) (string?)
+#{
+  \lyricmode {
+    \tag #'up { \markup { #(string-capitalize word) } }
+    \tag #'down { \markup { #(string-downcase word) } }
+  }
+#})

break_paged_output.ily

@@ -0,0 +1,2 @@
+\include "lilypond-book-preamble.ly"
+#(ly:set-option 'separate-page-formats "pdf")

chord_settings.ly

@@ -138,3 +138,33 @@ generalLayout = \layout {
           }
         }
      #}))
+
+% If you add this engraver to ChordNames Context chords get only printed on chordchanges and if its the first chord after line break.
+Ensure_first_chord_after_line_break_printed_engraver =
+  #(lambda (ctx)
+     (define last-system-column-rank 0)
+     (make-engraver
+      (acknowledgers
+       ((chord-name-interface this-engraver grob source-engraver)
+          (ly:grob-set-property! grob 'after-line-breaking
+            (lambda (grob)
+              (let ((current-system-column-rank (car (ly:grob-spanned-column-rank-interval (ly:grob-system grob)))))
+                (if (and
+                       (ly:context-property ctx 'chordChanges #f)
+                       (ly:grob-property grob 'begin-of-line-visible #f)
+                       (not (= last-system-column-rank current-system-column-rank)))
+                  ; the current chord handling implementation in lilypond uses 'begin-of-line-visible to mark repeated chords
+                  (ly:grob-set-property! grob 'begin-of-line-visible #f))
+                (set! last-system-column-rank current-system-column-rank)
+                (ly:chord-name::after-line-breaking grob)
+                )))))))
+
+% If you add this engraver to ChordNames Context chords get only printed on chordchanges and not at newline.
+Ignoring_newline_chord_changes_engraver =
+  #(lambda (ctx)
+     (make-engraver
+      (acknowledgers
+       ((chord-name-interface this-engraver grob source-engraver)
+         (when (and (ly:context-property ctx 'chordChanges #f) (ly:grob-property grob 'begin-of-line-visible #f))
+                  (ly:grob-suicide! grob)
+                )))))

footer_with_songinfo.ly

@@ -1,4 +1,27 @@
 #(use-modules (ice-9 receive))
+#(define (format-author layout authorId noDetails)
+  (let ((author (if (defined? 'AUTHOR_DATA) (assoc-ref AUTHOR_DATA authorId) #f)))
+    (if author
+      ((ly:output-def-lookup layout 'authorFormat)
+        noDetails
+        (assoc-ref author "name")
+        (assoc-ref author "trail_name")
+        (assoc-ref author "birth_year")
+        (assoc-ref author "death_year")
+        (assoc-ref author "organization")
+      )
+      "unbekannt")))
+
+#(define (find-author-ids-by contributionType authors)
+  (filter-map (lambda (authordata) (if (member contributionType (cdr authordata)) (car authordata) #f)) authors))
+
+#(define (find-author-id-with-part-numbers contributionType authors)
+  (filter-map (lambda (authordata)
+    (let ((contributionNumbers (filter-map (lambda (contribution) (if (and (list? contribution) (equal? contributionType (car contribution))) (cadr contribution) #f)) (cdr authordata)))
+          (authorId (car authordata)))
+      (if (null? contributionNumbers) #f (cons authorId contributionNumbers))
+    )) authors))
+
 #(define-markup-command (print-songinfo layout props) ()
   (define (songinfo-from songId key)
     (let ((song (if (defined? 'SONG_DATA) (assoc-ref SONG_DATA songId) #f)))
@@ -6,39 +29,14 @@
         (assoc-ref song key)
         (ly:warning (ly:format "song with id ~a not found" songId)))))
 
-  (define (format-author authorId noDetails)
-    (let ((author (if (defined? 'AUTHOR_DATA) (assoc-ref AUTHOR_DATA authorId) #f)))
-      (if author
-        ((ly:output-def-lookup layout 'authorFormat)
-          noDetails
-          (assoc-ref author "name")
-          (assoc-ref author "trail_name")
-          (assoc-ref author "birth_year")
-          (assoc-ref author "death_year")
-          (assoc-ref author "organization")
-        )
-        "unbekannt")))
-
   (define (format-poet poetId)
-    (string-append (ly:output-def-lookup layout 'poetPrefix) " " (format-author poetId #f)))
+    (string-append (ly:output-def-lookup layout 'poetPrefix) " " (format-author layout poetId #f)))
 
   (define (format-composer composerId)
-    (string-append (ly:output-def-lookup layout 'composerPrefix) " " (format-author composerId #f)))
+    (string-append (ly:output-def-lookup layout 'composerPrefix) " " (format-author layout composerId #f)))
 
   (define (format-poet-and-composer authorId)
-    (string-append (ly:output-def-lookup layout 'poetAndComposerEqualPrefix) " " (format-author authorId #f)))
+    (string-append (ly:output-def-lookup layout 'poetAndComposerEqualPrefix) " " (format-author layout authorId #f)))
-
-  (define (find-author-ids-by contributionType authors)
-    (filter-map (lambda (authordata) (if (member contributionType (cdr authordata)) (car authordata) #f)) authors)
-  )
-
-  (define (find-author-id-with-part-numbers contributionType authors)
-    (filter-map (lambda (authordata)
-      (let ((contributionNumbers (filter-map (lambda (contribution) (if (and (list? contribution) (equal? contributionType (car contribution))) (cadr contribution) #f)) (cdr authordata)))
-            (authorId (car authordata)))
-        (if (null? contributionNumbers) #f (cons authorId contributionNumbers))
-      )) authors)
-  )
 
   (define (numbered-contribution-prefix contributionNumbers prefixLookup)
     (string-append
@@ -53,6 +51,7 @@
   (define (format-authors authorIds)
     (map (lambda (authorId)
       (format-author
+        layout
         authorId
         (if (member authorId referencedAuthors)
           #t

scm/resolve_inherits.scm

@@ -0,0 +1,22 @@
+(define (resolve-inherit-entry-in-list alist entry)
+  (let* ((key (car entry))
+        (attributes (cdr entry))
+        (inherits (assoc-ref attributes "inherits")))
+    (if inherits
+      (let* ((alist-without-entry (alist-delete key alist))
+            (inherit-entry (assoc inherits alist-without-entry))
+            (inherits-attributes (if inherit-entry (alist-copy (cdr (resolve-inherit-entry-in-list alist-without-entry inherit-entry)))))
+            (override-attributes (alist-delete "inherits" attributes)))
+        (if inherit-entry
+          (begin
+            (for-each (lambda (attribute) (assoc-set! inherits-attributes (car attribute) (cdr attribute))) override-attributes)
+            (cons key inherits-attributes)
+          )
+          (ly:error "~a can not inherit from ~a" key inherits))
+      )
+      entry
+    )))
+
+(define (resolve-inherits alist)
+  (map (lambda (entry) (resolve-inherit-entry-in-list alist entry)) alist)
+)

scm/yaml_parser.scm

@@ -0,0 +1,10 @@
+(use-modules (ice-9 popen) (ice-9 textual-ports) (json parser))
+
+(define (yml-file->scm filename)
+  (let* ((python_cmd (string-append "import sys, yaml, json; print(json.dumps(yaml.safe_load(open('" filename "'))))"))
+          (pipe (open-pipe* OPEN_READ "python3" "-c" python_cmd))
+          (json (get-string-all pipe)))
+    (close-pipe pipe)
+    (json-string->scm json)))
+
+(define (parse-yml-file filename) (resolve-inherits (yml-file->scm filename)))

snippet_include.ly

@@ -3,6 +3,3 @@
 #(define noDefaultOutput (if (defined? 'noDefaultOutput) noDefaultOutput #f))
 
 \include #(if noDefaultOutput "void.ly" "all_base_includes.ly")
-
-#(define AUTHOR_DATA (if (defined? 'AUTHOR_DATA) AUTHOR_DATA (call-with-input-file "../data/authors.json" json->scm)))
-#(define SONG_DATA (if (defined? 'SONG_DATA) SONG_DATA (call-with-input-file "../data/songs.json" json->scm)))

standalone_output.ly

@@ -18,19 +18,19 @@ TEXT = \markuplist {
   \TEXT
 }
 
-% nur Output wenn noStandaloneOutput auf false steht
+#(if (not noStandaloneOutput)
-output = #(if (not noStandaloneOutput)
+(begin
-#{
+  (let ((header (ly:book-header HEADER)) (paper (ly:book-paper HEADER)))
-  \bookpart {
+    (if header (set! $defaultheader header))
-    \HEADER
+    (if paper (set! $defaultpaper paper))
-
+  )
+  (add-score #{
     \score {
       \MUSIC
       \layout { \LAYOUT }
-    }
+    }#})
-
+  (add-score TEXT)
-    \TEXT
+  (add-score #{
-
     \score {
       \unfoldRepeats { \MUSIC \INLINESCOREMUSIC }
       \midi {
@@ -40,15 +40,5 @@ output = #(if (not noStandaloneOutput)
           tempoWholesPerMinute = #(ly:make-moment midiQuarterNoteSpeed 4)
         }
       }
-    }
+    }#})
-
+))
-
-  }
-#}
-)
-
-% if we don't want a standalone output, cause we compile a book, we just have an empty output here,
-% so lilypond does not generate output for this song
-\book {
-  \bookpart { \output }
-}

swing_style.ly

@@ -45,4 +45,28 @@ swing = \mark \markup {
       }
     }
   }
-}
+}
+
+\include "swing.ly"
+
+swingMusic =
+#(define-music-function (parser location music) (ly:music?)
+  (define (partial-duration-length m)
+    (let ((name (ly:music-property m 'name))
+          (es (ly:music-property m 'elements))
+          (e (ly:music-property m 'element)))
+      (if (pair? es)
+        (partial-duration-length (car es))
+        (if (ly:music? e)
+          (if (and (eq? name 'ContextSpeccedMusic) (eq? (ly:music-property e 'name) 'PartialSet))
+            (ly:duration-length (ly:music-property e 'duration))
+            (if (eq? name 'NoteEvent)
+              ZERO-MOMENT
+              (partial-duration-length e)
+            )
+          )
+          ZERO-MOMENT))))
+#{
+  \swing
+  \applySwingWithOffset 8 #'(2 1) #(partial-duration-length music) #music
+#})

title_with_category_images.ly

@@ -48,8 +48,9 @@
 
 #(define-markup-command (title-with-category-images layout props right)(boolean?)
    (interpret-markup layout props
-     (let* ((title (chain-assoc-get 'header:title props #f))
+     (let* ((title (chain-assoc-get 'header:title props ""))
-            (pdfbookmark (chain-assoc-get 'header:songfilename props title)))
+            (starttext (chain-assoc-get 'header:starttext props #f))
+            (pdfbookmark (if starttext (string-append starttext " | " title) title)))
      (if title
      ;(if (chain-assoc-get 'header:categories props #f)
        (if right

toc_include.ly

@@ -219,29 +219,152 @@ indexSection =
    "Add a section line to the alphabetical index, using @code{indexSectionMarkup} paper variable markup. This can be used to divide the alphabetical index into different sections, for example one section for each first letter."
    (add-index-item! 'indexSectionMarkup text sorttext))
 
-#(define (extract-var-from-module module sym)
+#(define (extract-and-check-vars-from-header bookheader varlist)
-  (let ((variableref (assoc-ref module sym)))
+  (let* ((headervars (hash-map->list cons (struct-ref (ly:book-header bookheader) 0)))
-    (if variableref (variable-ref variableref) #f))
+         (extract-var-and-check (lambda (headervar)
-  )
+           (let* ((variableref (assoc-ref headervars headervar))
+                  (extracted (if variableref (variable-ref variableref) #f)))
+              (if (and extracted (not (and (string? extracted) (string-null? extracted)))) extracted #f)))))
+    (map (lambda (varsymbol)
+      (cons varsymbol (extract-var-and-check varsymbol))
+    ) varlist)))
 
 headerToTOC = #(define-music-function (parser location header label) (ly:book? symbol?)
     (let*
       (
-       (headervars (hash-map->list cons (struct-ref (ly:book-header header) 0)))
+       (extractedheadervars (extract-and-check-vars-from-header header '(title starttext alttitle altalttitle categorytitle categories)))
-       (extract-var-and-check (lambda (headervar) (let
+       (title (assq-ref extractedheadervars 'title))
-                                                   ((extracted (extract-var-from-module headervars headervar)))
+       (starttext (assq-ref extractedheadervars 'starttext))
-                                                   (if (and extracted (not (string-null? extracted))) extracted #f)
+       (alttitle (assq-ref extractedheadervars 'alttitle))
-                                                    )))
+       (altalttitle (assq-ref extractedheadervars 'altalttitle))
-       (title (extract-var-and-check 'title))
+       (categorytitle (assq-ref extractedheadervars 'categorytitle))
-       (alttitle (extract-var-and-check 'alttitle))
+       (categories (assq-ref extractedheadervars 'categories))
-       (altalttitle (extract-var-and-check 'altalttitle))
-       (categorytitle (extract-var-and-check 'categorytitle))
-       (categories (extract-var-and-check 'categories))
        (add-to-toc! (lambda (toctitle tocmarkup)
                       (add-index-item! 'indexItemMarkup tocmarkup toctitle label)))
        )
            (if categories (add-category-index-item! (string-tokenize categories) 'indexItemMarkup (cons (list (cons 'rawtext (if categorytitle categorytitle title))) '()) label))
+           (if starttext (add-to-toc! starttext (cons (list (cons 'rawtext starttext) (cons 'alternative #t)) '())))
            (if alttitle (add-to-toc! alttitle (cons (list (cons 'rawtext alttitle) (cons 'alternative #t)) '())))
            (if altalttitle (add-to-toc! altalttitle (cons (list (cons 'rawtext altalttitle) (cons 'alternative #t)) '())))
            (if title (add-to-toc! title (cons (list (cons 'rawtext title)) '())) #{ #})
     ))
+
+
+%% https://github.com/NalaGinrut/guile-csv/blob/master/csv/csv.scm
+
+#(define* (sxml->csv sxml port #:key (delimiter #\,))
+  (let* ((d (string delimiter))
+         (csv (map (lambda (l) (string-join l d)) sxml)))
+    (for-each (lambda (l)
+                (format port "~a~%" l))
+              csv)))
+
+#(define csv-write sxml->csv)
+
+#(define-markup-command (write-toc-csv layout props) ()
+  (define (csv-escape field)
+    (if (string-null? field)
+        field
+        (string-append
+          "\""
+          (ly:string-substitute "\n" "\\n"
+            (ly:string-substitute "\"" "\\\"" field))
+          "\"")))
+  (define (format-authors authorIds)
+    (string-join (map (lambda (authorId) (format-author layout authorId #f)) authorIds) ", "))
+  (define cr-regex (ly:make-regex "\r"))
+  (define crlf-regex (ly:make-regex "\r\n"))
+  (define para-sep-regex (ly:make-regex "\n[ \t\n]*\n[ \t\n]*"))
+  (define whitespace-regex (ly:make-regex "[ \t\n]+"))
+  (define leading-whitespace-regex (ly:make-regex "^[ \t\n]+"))
+  (define trailing-whitespace-regex (ly:make-regex "[ \t\n]+$"))
+  (define (cleanup-whitespaces str)
+    (ly:regex-replace leading-whitespace-regex
+      (ly:regex-replace trailing-whitespace-regex
+        (ly:regex-replace whitespace-regex str " ")
+        "")
+      ""))
+  (define (format-info-paragraphs text)
+    (let* ((para-strings (ly:regex-split
+                        para-sep-regex
+                        (ly:regex-replace
+                         cr-regex
+                         (ly:regex-replace crlf-regex text "\n")
+                         "\n")))
+         (para-lines (map cleanup-whitespaces para-strings)))
+    (string-join para-lines "\n")))
+  (define (generate-toc-csv labelPageTable)
+    (let ((song-lines (map (lambda (song)
+            (let* ((filename (symbol->string (car song)))
+                   (songvars (cdr song))
+                   (page-number (number->string (assoc-get (assq-ref songvars 'label) labelPageTable)))
+                   (extractedheadervars (extract-and-check-vars-from-header (assq-ref songvars 'header)
+                      '(title starttext alttitle altalttitle categorytitle categories authors year_text year_melody year_translation year_composition infotext translation pronunciation copyright source)))
+                   (headervar-or-empty (lambda (varsym)
+                     (let ((extracted (assq-ref extractedheadervars varsym)))
+                       (if extracted extracted ""))))
+                   (authors (assq-ref extractedheadervars 'authors))
+                   (poetIds (find-author-ids-by 'text authors))
+                   (translatorIds (find-author-ids-by 'translation authors))
+                   (versePoetData (find-author-id-with-part-numbers 'verse authors))
+                   (composerIds (find-author-ids-by 'melody authors))
+                   (verseComposerData (find-author-id-with-part-numbers 'meloverse authors))
+                   (voiceComposerData (find-author-id-with-part-numbers 'voice authors))
+                   (compositionIds (find-author-ids-by 'composition authors))
+                   (bridgeIds (find-author-ids-by 'bridge authors))
+                   (interludeIds (find-author-ids-by 'interlude authors)))
+              (map csv-escape
+                (list
+                  filename
+                  page-number
+                  (headervar-or-empty 'title)
+                  (headervar-or-empty 'starttext)
+                  (headervar-or-empty 'alttitle)
+                  (headervar-or-empty 'altalttitle)
+                  (headervar-or-empty 'categorytitle)
+                  (headervar-or-empty 'categories)
+                  (format-authors (append poetIds (map car versePoetData)))
+                  (format-authors translatorIds)
+                  (format-authors (append composerIds compositionIds bridgeIds interludeIds (map car voiceComposerData) (map car verseComposerData)))
+                  (headervar-or-empty 'year_text)
+                  (headervar-or-empty 'year_melody)
+                  (headervar-or-empty 'year_translation)
+                  (headervar-or-empty 'year_composition)
+                  (headervar-or-empty 'copyright)
+                  (headervar-or-empty 'source)
+                  (format-info-paragraphs (headervar-or-empty 'infotext))
+                  (format-info-paragraphs (headervar-or-empty 'translation))
+                  (format-info-paragraphs (headervar-or-empty 'pronunciation))
+              ))))
+            (alist-delete 'imagePage (alist-delete 'emptyPage song-list)))))
+    (call-with-output-file "toc.csv"
+      (lambda (port)
+       (csv-write (cons '(
+        "filename"
+        "page-number"
+        "title"
+        "starttext"
+        "alttitle"
+        "altalttitle"
+        "categorytitle"
+        "categories"
+        "poets"
+        "translators"
+        "composers"
+        "year_text"
+        "year_melody"
+        "year_translation"
+        "year_composition"
+        "copyright"
+        "source"
+        "infotext"
+        "translation"
+        "pronunciation"
+        ) song-lines) port))
+  )))
+  ; we use a delayed stencil to have all the page references available
+  (ly:make-stencil
+    `(delay-stencil-evaluation
+      ,(delay (let* ((table (ly:output-def-lookup layout 'label-page-table)))
+        (generate-toc-csv (if (list? table) table '()))
+        empty-stencil)))))

verses_with_chords.ly

@@ -277,6 +277,7 @@
               \ChordNames
               \override VerticalAxisGroup.staff-affinity = ##f
               \override ChordName.extra-spacing-width = #'(-0.1 . 0.1)
+              \consists \Ensure_first_chord_after_line_break_printed_engraver
             }
             \context {
               \Score