From 384fa463d71c6a7ecd24befd5670376874ce465f Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Tue, 11 Jun 2024 23:56:45 +0300 Subject: [PATCH 01/32] feat(tooling): Add developer target to build Rust API docs --- Makefile.am | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 7ea15a56b..36510d8c1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -196,7 +196,7 @@ selfcheck: | $(bin_PROGRAMS) $(_BUILT_SUBDIRS) $(PDFINFO) $$output | $(GREP) "SILE v$(VERSION)" .PHONY: docs -docs: $(_MANUAL) lua-api-docs +docs: $(_MANUAL) lua-api-docs rust-api-docs .PHONY: lua-api-docs lua-api-docs: lua-api-docs/index.html @@ -204,6 +204,12 @@ lua-api-docs: lua-api-docs/index.html lua-api-docs/index.html: build-aux/config.ld $(LDOC) -c build-aux/config.ld . +.PHONY: rust-api-docs +rust-api-docs: rust-api-docs/doc/sile/index.html + +rust-api-docs/doc/sile/index.html: + cargo doc $(CARGO_FEATURE_ARGS) --no-deps --target-dir rust-api-docs + .PHONY: docs-figures docs-figures: $(FIGURES) From 8887271be5db8c42bbccb9b1b3cb814eb4393c62 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 12 Jun 2024 01:59:37 +0300 Subject: [PATCH 02/32] style(classes): Restyle with stylua --- classes/plain.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/classes/plain.lua b/classes/plain.lua index 149694153..67d681305 100644 --- a/classes/plain.lua +++ b/classes/plain.lua @@ -87,8 +87,7 @@ function class:registerCommands () an input document this is typically done with an empty line between paragraphs, but calling the \par command explicitly or from Lua code running SILE.call("par") will end the current paragraph. -]] - ) +]]) end SILE.settings:set("current.parindent", SILE.types.node.glue()) SILE.process(content) From d42ef0c0f6a11616047aca2e08bf6ba851da2001 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 12 Jun 2024 13:01:54 +0300 Subject: [PATCH 03/32] test(packages): Add test for language scope leak in bibliography --- tests/bug-2054.bib | 10 ++++ tests/bug-2054.expected | 114 ++++++++++++++++++++++++++++++++++++++++ tests/bug-2054.sil | 15 ++++++ 3 files changed, 139 insertions(+) create mode 100644 tests/bug-2054.bib create mode 100644 tests/bug-2054.expected create mode 100644 tests/bug-2054.sil diff --git a/tests/bug-2054.bib b/tests/bug-2054.bib new file mode 100644 index 000000000..27b494cac --- /dev/null +++ b/tests/bug-2054.bib @@ -0,0 +1,10 @@ +@phdthesis{foo, + author = {Doe, John and Smith, Jane and Bob and Alice}, + title = {A Tale of Four Authors}, +} + +@book{bar, + author = {Doe, John and Smith, Jane}, + translator = {Pumpernickel, Peter}, + title = {A Book}, +} diff --git a/tests/bug-2054.expected b/tests/bug-2054.expected new file mode 100644 index 000000000..f77f64d7f --- /dev/null +++ b/tests/bug-2054.expected @@ -0,0 +1,114 @@ +Set paper size 297.6377985 419.5275636 +Begin page +Mx 14.8819 +My 28.5447 +Set font Gentium Plus;10;400;;normal;;;LTR +T 39 82 72 w=15.5762 (Doe) +Mx 30.4581 +T 15 w=2.2900 (,) +Mx 34.9513 +T 45 w=3.0518 (J) +Mx 38.0031 +T 17 w=2.2900 (.) +Mx 42.4963 +T 68 81 71 w=15.3076 (and) +Mx 60.0072 +T 54 80 76 87 75 w=24.4629 (Smith) +Mx 84.4701 +T 15 w=2.2900 (,) +Mx 88.9633 +T 45 w=3.0518 (J) +Mx 92.0151 +T 17 w=2.2900 (.) +Mx 94.3051 +T 15 w=2.2900 (,) +Mx 98.7984 +Set font Gentium Plus;10;400;Italic;normal;;;LTR +T 36 w=5.6006 (A) +Mx 106.4019 +T 37 82 82 78 w=17.7783 (Book) +Mx 124.1802 +Set font Gentium Plus;10;400;;normal;;;LTR +T 17 w=2.2900 (.) +Mx 128.6735 +T 55 85 68 81 86 79 68 87 72 71 w=43.9990 (Translated) +Mx 174.8758 +T 69 92 w=10.0098 (by) +Mx 187.0887 +T 51 88 80 83 72 85 81 76 70 78 72 79 w=57.2412 (Pumpernickel) +Mx 244.3300 +T 15 w=2.2900 (,) +Mx 248.8232 +T 51 72 87 72 85 w=21.8701 (Peter) +Mx 270.6933 +T 17 w=2.2900 (.) +Mx 14.8819 +My 40.5447 +T 39 82 72 w=15.5762 (Doe) +Mx 32.6737 +T 72 87 w=8.0615 (et) +Mx 42.9508 +T 68 79 w=7.2998 (al) +Mx 50.2506 +T 17 w=2.2900 (.) +Mx 52.5406 +T 15 w=2.2900 (,) +Mx 57.0463 +T 180 w=4.3604 (“) +Mx 61.4066 +T 36 w=5.9521 (A) +Mx 69.5744 +T 55 68 79 72 w=17.4268 (Tale) +Mx 89.2167 +T 82 73 w=8.1396 (of) +Mx 99.5720 +T 41 82 88 85 w=19.0674 (Four) +Mx 120.8550 +T 36 88 87 75 82 85 86 w=33.0615 (Authors) +Mx 153.9165 +T 17 w=2.2900 (.) +Mx 156.2065 +T 181 w=4.3604 (”) +Mx 14.8819 +My 52.5447 +T 39 82 72 w=15.5762 (Doe) +Mx 30.4581 +T 15 w=2.2900 (,) +Mx 34.9513 +T 45 w=3.0518 (J) +Mx 38.0031 +T 17 w=2.2900 (.) +Mx 42.4963 +T 68 81 71 w=15.3076 (and) +Mx 60.0072 +T 54 80 76 87 75 w=24.4629 (Smith) +Mx 84.4701 +T 15 w=2.2900 (,) +Mx 88.9633 +T 45 w=3.0518 (J) +Mx 92.0151 +T 17 w=2.2900 (.) +Mx 94.3051 +T 15 w=2.2900 (,) +Mx 98.7984 +Set font Gentium Plus;10;400;Italic;normal;;;LTR +T 36 w=5.6006 (A) +Mx 106.4019 +T 37 82 82 78 w=17.7783 (Book) +Mx 124.1802 +Set font Gentium Plus;10;400;;normal;;;LTR +T 17 w=2.2900 (.) +Mx 128.6735 +T 55 85 68 81 86 79 68 87 72 71 w=43.9990 (Translated) +Mx 174.8758 +T 69 92 w=10.0098 (by) +Mx 187.0887 +T 51 88 80 83 72 85 81 76 70 78 72 79 w=57.2412 (Pumpernickel) +Mx 244.3300 +T 15 w=2.2900 (,) +Mx 248.8232 +T 51 72 87 72 85 w=21.8701 (Peter) +Mx 270.6933 +T 17 w=2.2900 (.) +End page +Finish diff --git a/tests/bug-2054.sil b/tests/bug-2054.sil new file mode 100644 index 000000000..d8b381879 --- /dev/null +++ b/tests/bug-2054.sil @@ -0,0 +1,15 @@ +\begin[papersize=a6]{document} +\nofolios +\neverindent +\language[main=en] +\use[module=packages.bibtex] +\loadbibliography[file=tests/bug-2054.bib] + +\reference{bar} + +\reference{foo} + +% Should not use Latin localization, should match first use of reference +\reference{bar} + +\end{document} From 283fdc3062313394850cf5067c846db82f264c2e Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 12 Jun 2024 13:30:23 +0300 Subject: [PATCH 04/32] fix(languages): Always set Fluent locale when setting `document.language` --- core/languages.lua | 12 ++++++++++-- core/settings.lua | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/core/languages.lua b/core/languages.lua index b5c0f59a3..40695683c 100644 --- a/core/languages.lua +++ b/core/languages.lua @@ -33,6 +33,10 @@ SILE.languageSupport = { end local ftlresource = string.format("i18n.%s", language) SU.debug("fluent", "Loading FTL resource", ftlresource, "into locale", language) + -- This needs to be set so that we load localizations into the right bundle, + -- but this breaks the sync enabled by the hook in the document.language + -- setting, so we want to set it back when we're done. + local original_language = fluent:get_locale() fluent:set_locale(language) local gotftl, ftl = pcall(require, ftlresource) if not gotftl then @@ -46,6 +50,7 @@ SILE.languageSupport = { if type(lang) == "table" and lang.init then lang.init() end + fluent:set_locale(original_language) end, } @@ -65,10 +70,11 @@ end, nil, nil, true) SILE.registerCommand("fluent", function (options, content) local key = content[1] local locale = options.locale or SILE.settings:get("document.language") + local original_locale = fluent:get_locale() + fluent:set_locale(locale) SU.debug("fluent", "Looking for", key, "in", locale) local entry if key then - fluent:set_locale(locale) entry = fluent:get_message(key) else SU.warn("Fluent localization function called without passing a valid message id") @@ -83,12 +89,13 @@ SILE.registerCommand("fluent", function (options, content) if entry then message = entry:format(options) end - fluent:set_locale(locale) end + fluent:set_locale(original_locale) SILE.processString(("%s"):format(message), "xml") end, nil, nil, true) SILE.registerCommand("ftl", function (options, content) + local original_locale = fluent:get_locale() local locale = options.locale or SILE.settings:get("document.language") SU.debug("fluent", "Loading message(s) into locale", locale) fluent:set_locale(locale) @@ -98,6 +105,7 @@ SILE.registerCommand("ftl", function (options, content) local input = content[1] fluent:add_messages(input, locale) end + fluent:set_locale(original_locale) end, nil, nil, true) require("languages.unicode") diff --git a/core/settings.lua b/core/settings.lua index 0d1acea0c..668340024 100644 --- a/core/settings.lua +++ b/core/settings.lua @@ -19,6 +19,9 @@ function settings:_init () parameter = "document.language", type = "string", default = "en", + hook = function (language) + fluent:set_locale(language) + end, help = "Locale for localized language support", }) From 671c5bd373eccf7ebd38bde64f0a99152fb532cd Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 12 Jun 2024 13:34:34 +0300 Subject: [PATCH 05/32] refactor(languages): Rely on language setting hook to set Fluent locale --- core/font.lua | 3 +-- spec/hyphenator_spec.lua | 9 +++------ spec/languages_spec.lua | 10 +++------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/core/font.lua b/core/font.lua index 1b673015c..21cb2caee 100644 --- a/core/font.lua +++ b/core/font.lua @@ -48,9 +48,8 @@ SILE.registerCommand("font", function (options, content) -- end options.language = newlang end - SILE.settings:set("document.language", options.language) - fluent:set_locale(options.language) SILE.languageSupport.loadLanguage(options.language) + SILE.settings:set("document.language", options.language) end if options.script then SILE.settings:set("font.script", options.script) diff --git a/spec/hyphenator_spec.lua b/spec/hyphenator_spec.lua index e41b37297..a4a4531bd 100644 --- a/spec/hyphenator_spec.lua +++ b/spec/hyphenator_spec.lua @@ -6,9 +6,9 @@ SILE.init() describe("Hyphenation module", function () local hyphenate = SILE.showHyphenationPoints + SILE.call("language", { main = "fr" }) + describe("minWord with UTF8 in input text", function () - SILE.languageSupport.loadLanguage("fr") - fluent:set_locale("fr") -- Trigger the initialization of the hyphenator -- so SILE._hyphenators["fr"] is created hyphenate("série", "fr") @@ -33,10 +33,7 @@ describe("Hyphenation module", function () end) describe("exceptions with UTF8 in input text", function () - SILE.languageSupport.loadLanguage("fr") - fluent:set_locale("fr") - - SILE.call("hyphenator:add-exceptions", { lang = "fr" }, { "légè-rement" }) + SILE.call("hyphenator:add-exceptions", {}, { "légè-rement" }) it("should hyphenate with exception rule", function () assert.is.equal("légè-rement", hyphenate("légèrement", "fr")) diff --git a/spec/languages_spec.lua b/spec/languages_spec.lua index 1568e4f40..2264e9470 100644 --- a/spec/languages_spec.lua +++ b/spec/languages_spec.lua @@ -4,8 +4,7 @@ describe("Language module", function () describe("Norwegian", function () local hyphenate = SILE.showHyphenationPoints - SILE.languageSupport.loadLanguage("no") - fluent:set_locale("no") + SILE.call("language", { main = "no" }) it("should hyphenate", function () assert.is.equal("Nor-we-gian", hyphenate("Norwegian", "no")) @@ -13,14 +12,12 @@ describe("Language module", function () end) it("should have localizations", function () - fluent:set_locale("no") local hello = fluent:get_message("hello")({ name = "Busted" }) assert.is.equal("Hei Busted!", hello) end) describe("Norwegian Bokmål", function () - SILE.languageSupport.loadLanguage("nb") - fluent:set_locale("nb") + SILE.call("language", { main = "nb" }) it("should hyphenate", function () assert.is.equal("Nor-we-gian", hyphenate("Norwegian", "nb")) @@ -34,8 +31,7 @@ describe("Language module", function () end) describe("Norwegian Nynorsk", function () - SILE.languageSupport.loadLanguage("nn") - fluent:set_locale("nn") + SILE.call("language", { main = "nn" }) it("should hyphenate", function () assert.is.equal("Nor-we-gian", hyphenate("Norwegian", "nn")) From 83db3d94dd94eb949b96a961f9a821d0b9c3bf00 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 12 Jun 2024 14:17:37 +0300 Subject: [PATCH 06/32] chore(languages): Warn when setting a language that has not been loaded --- core/languages.lua | 6 +++--- core/settings.lua | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/languages.lua b/core/languages.lua index 40695683c..c2b262176 100644 --- a/core/languages.lua +++ b/core/languages.lua @@ -10,17 +10,17 @@ loadkit.register("ftl", function (file) return assert(fluent:add_messages(contents)) end) -local loadonce = {} +SILE.scratch.loaded_languages = {} SILE.languageSupport = { languages = {}, loadLanguage = function (language) language = language or SILE.settings:get("document.language") language = cldr.locales[language] and language or "und" - if loadonce[language] then + if SILE.scratch.loaded_languages[language] then return end - loadonce[language] = true + SILE.scratch.loaded_languages[language] = true local langresource = string.format("languages.%s", language) local gotlang, lang = pcall(require, langresource) if not gotlang then diff --git a/core/settings.lua b/core/settings.lua index 668340024..c5e8a45e0 100644 --- a/core/settings.lua +++ b/core/settings.lua @@ -20,6 +20,13 @@ function settings:_init () type = "string", default = "en", hook = function (language) + if SILE.scratch.loaded_languages and not SILE.scratch.loaded_languages[language] then + SU.warn(([[Setting document.language to '%s', but support for '%s' has not been loaded! + + Consider invoking \language[main=%s] which loads language support before + setting it or manually calling SILE.languageSupport.loadLanguage("%s"). + ]]):format(language, language, language, language)) + end fluent:set_locale(language) end, help = "Locale for localized language support", From 70b329ebac62e7267e0c2dd9cb842d2cfa112d8b Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sat, 15 Jun 2024 13:36:11 +0300 Subject: [PATCH 07/32] chore(packages): Improve error message when a font doesn't support math (#2070) --- packages/math/base-elements.lua | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/math/base-elements.lua b/packages/math/base-elements.lua index df3ba44e1..aa2dfd590 100644 --- a/packages/math/base-elements.lua +++ b/packages/math/base-elements.lua @@ -115,11 +115,18 @@ local function retrieveMathTable (font) if not face then SU.error("Could not find requested font " .. font .. " or any suitable substitutes") end - local mathTable = ot.parseMath(hb.get_table(face, "MATH")) - local upem = ot.parseHead(hb.get_table(face, "head")).unitsPerEm - if mathTable == nil then - SU.error("You must use a math font for math rendering.") + local fontHasMathTable, rawMathTable, mathTableParsable, mathTable + fontHasMathTable, rawMathTable = pcall(hb.get_table, face, "MATH") + if fontHasMathTable then + mathTableParsable, mathTable = pcall(ot.parseMath, rawMathTable) + end + if not fontHasMathTable or not mathTableParsable then + SU.error(([[You must use a math font for math rendering. + + The math table in '%s' could not be %s. + ]]):format(face.filename, fontHasMathTable and "parsed" or "loaded")) end + local upem = ot.parseHead(hb.get_table(face, "head")).unitsPerEm local constants = {} for k, v in pairs(mathTable.mathConstants) do if type(v) == "table" then From 895c575e55974e4485701e04256516d130cc66f8 Mon Sep 17 00:00:00 2001 From: jodros <15092760+jodros@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:43:11 -0300 Subject: [PATCH 08/32] feat(languages): Add Brazilian Portuguese specific hyphenation points --- languages/pt.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/languages/pt.lua b/languages/pt.lua index 4368e4816..cf9fc09a7 100644 --- a/languages/pt.lua +++ b/languages/pt.lua @@ -4,6 +4,7 @@ SILE.nodeMakers.pt = pl.class(SILE.nodeMakers.unicode) SILE.nodeMakers.pt.handleWordBreak = SILE.nodeMakers.unicode._handleWordBreakRepeatHyphen SILE.nodeMakers.pt.handlelineBreak = SILE.nodeMakers.unicode._handlelineBreakRepeatHyphen +-- further patterns "1nô", "1tô" and "1cô" were added for Brazilian Portuguese SILE.hyphenator.languages["pt"] = {} SILE.hyphenator.languages["pt"].patterns = { "1b2l", @@ -32,6 +33,7 @@ SILE.hyphenator.languages["pt"].patterns = { "1cu", "1cá", "1câ", + "1cô", "1cã", "1cé", "1cí", @@ -172,6 +174,7 @@ SILE.hyphenator.languages["pt"].patterns = { "1nu", "1ná", "1nâ", + "1nô", "1nã", "1né", "1ní", @@ -236,6 +239,7 @@ SILE.hyphenator.languages["pt"].patterns = { "1tu", "1tá", "1tâ", + "1tô", "1tã", "1té", "1tí", From c4ae919ed72bce5740cb59809be8557dfe9b9321 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sat, 8 Jun 2024 09:18:35 +0200 Subject: [PATCH 09/32] feat(packages): Support crossref field in bibTeX bibliography Closes #2021 --- packages/bibtex/bibliography.lua | 2 ++ packages/bibtex/init.lua | 46 ++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/packages/bibtex/bibliography.lua b/packages/bibtex/bibliography.lua index 818e267be..4a0fc646e 100644 --- a/packages/bibtex/bibliography.lua +++ b/packages/bibtex/bibliography.lua @@ -286,6 +286,7 @@ Bibliography = { produceCitation = function (cite, bib, style) local item = bib[cite.key] if not item then + -- Should have been already checked by the caller return Bibliography.Errors.UNKNOWN_REFERENCE end local t = Bibliography.buildEnv(cite, item.attributes, style) @@ -296,6 +297,7 @@ Bibliography = { produceReference = function (cite, bib, style) local item = bib[cite.key] if not item then + -- Should have been already checked by the caller return Bibliography.Errors.UNKNOWN_REFERENCE end item.type = item.type:gsub("^%l", string.upper) diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index 4bf831bf7..190b9f16a 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -58,6 +58,34 @@ local parseBibtex = function (fn) return entries end +--- Resolve the 'crossref' field on a bibliography entry. +-- (Supplementing the entry with the attributes of the parent entry.) +-- Once resolved recursively, the crossref field is removed from the entry. +-- So this is intended to be called at first use of the entry, and have no +-- effect on subsequent uses: BibTeX does seem to mandate crossref to be +-- defined before the entry that uses it, or even in the same bibliography +-- file. +-- @tparam table bib Bibliography +-- @tparam string key Valid entry key +local function crossrefResolve (bib, key) + local entry = bib[key] + local crossref = entry.attributes.crossref + if crossref then + local parent = bib[crossref] + entry.attributes.crossref = nil + if parent then + crossrefResolve(bib, crossref) + for k, v in pairs(parent.attributes) do + if not entry.attributes[k] then + entry.attributes[k] = v + end + end + else + SU.warn("Unknown crossref " .. crossref .. " in bibliography entry " .. key) + end + end +end + function package:_init () base._init(self) SILE.scratch.bibtex = { bib = {} } @@ -87,13 +115,14 @@ function package:registerCommands () if not options.key then options.key = SU.ast.contentToString(content) end - local style = SILE.settings:get("bibtex.style") - local bibstyle = require("packages.bibtex.styles." .. style) - local cite = Bibliography.produceCitation(options, SILE.scratch.bibtex.bib, bibstyle) - if cite == Bibliography.Errors.UNKNOWN_REFERENCE then + if not SILE.scratch.bibtex.bib[options.key] then SU.warn("Unknown reference in citation " .. options.key) return end + crossrefResolve(SILE.scratch.bibtex.bib, options.key) + local style = SILE.settings:get("bibtex.style") + local bibstyle = require("packages.bibtex.styles." .. style) + local cite = Bibliography.produceCitation(options, SILE.scratch.bibtex.bib, bibstyle) SILE.processString(("%s"):format(cite), "xml") end) @@ -101,13 +130,14 @@ function package:registerCommands () if not options.key then options.key = SU.ast.contentToString(content) end + if not SILE.scratch.bibtex.bib[options.key] then + SU.warn("Unknown reference in citation " .. options.key) + return + end + crossrefResolve(SILE.scratch.bibtex.bib, options.key) local style = SILE.settings:get("bibtex.style") local bibstyle = require("packages.bibtex.styles." .. style) local cite, err = Bibliography.produceReference(options, SILE.scratch.bibtex.bib, bibstyle) - if cite == Bibliography.Errors.UNKNOWN_REFERENCE then - SU.warn("Unknown reference in citation " .. tostring(options.key)) - return - end if cite == Bibliography.Errors.UNKNOWN_TYPE then SU.warn("Unknown type @" .. err .. " in citation for reference " .. options.key) return From ca906f0cfc882c5ad4ecb410f6aec001018b64a8 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sun, 9 Jun 2024 11:53:53 +0200 Subject: [PATCH 10/32] feat(packages): Support `@xdata` entry type and xdata field in bibTeX bibliography Note that we do not support here expanding "xdata=" contructs inside other fields i.e. the "granular" per-field inheritance supported by BibLaTeX and originally Biber, expanding xdata=key-field-index inside other field values, which requires list fields (e.g. author) to already have been split and parsed... --- packages/bibtex/init.lua | 57 ++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index 190b9f16a..6d3731b6c 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -58,30 +58,49 @@ local parseBibtex = function (fn) return entries end ---- Resolve the 'crossref' field on a bibliography entry. +--- Resolve the 'crossref' and 'xdata' fields on a bibliography entry. -- (Supplementing the entry with the attributes of the parent entry.) --- Once resolved recursively, the crossref field is removed from the entry. +-- Once resolved recursively, the crossref and xdata fields are removed +-- from the entry. -- So this is intended to be called at first use of the entry, and have no --- effect on subsequent uses: BibTeX does seem to mandate crossref to be --- defined before the entry that uses it, or even in the same bibliography +-- effect on subsequent uses: BibTeX does seem to mandate cross refererences +-- to be defined before the entry that uses it, or even in the same bibliography -- file. +-- Implementation note: +-- We are not here to check the consistency of the BibTeX file, so there is +-- no check that xdata refers only to @xdata entries +-- Removing the crossref field implies we won't track its use and implicitely +-- cite referenced entries in the bibliography over a certain threshold. -- @tparam table bib Bibliography --- @tparam string key Valid entry key -local function crossrefResolve (bib, key) - local entry = bib[key] +-- @tparam table entry Bibliography entry +local function crossrefAndXDataResolve (bib, entry) + local refs + local xdata = entry.attributes.xdata + if xdata then + refs = xdata and pl.stringx.split(xdata, ",") + entry.attributes.xdata = nil + end local crossref = entry.attributes.crossref if crossref then - local parent = bib[crossref] + refs = refs or {} + table.insert(refs, crossref) entry.attributes.crossref = nil + end + + if not refs then + return + end + for _, ref in ipairs(refs) do + local parent = bib[ref] if parent then - crossrefResolve(bib, crossref) + crossrefAndXDataResolve(bib, parent) for k, v in pairs(parent.attributes) do if not entry.attributes[k] then entry.attributes[k] = v end end else - SU.warn("Unknown crossref " .. crossref .. " in bibliography entry " .. key) + SU.warn("Unknown crossref " .. ref .. " in bibliography entry " .. entry.label) end end end @@ -115,11 +134,16 @@ function package:registerCommands () if not options.key then options.key = SU.ast.contentToString(content) end - if not SILE.scratch.bibtex.bib[options.key] then + local entry = SILE.scratch.bibtex.bib[options.key] + if not entry then SU.warn("Unknown reference in citation " .. options.key) return end - crossrefResolve(SILE.scratch.bibtex.bib, options.key) + if entry.type == "xdata" then + SU.warn("Skipped citation of @xdata entry " .. options.key) + return + end + crossrefAndXDataResolve(SILE.scratch.bibtex.bib, entry) local style = SILE.settings:get("bibtex.style") local bibstyle = require("packages.bibtex.styles." .. style) local cite = Bibliography.produceCitation(options, SILE.scratch.bibtex.bib, bibstyle) @@ -130,11 +154,16 @@ function package:registerCommands () if not options.key then options.key = SU.ast.contentToString(content) end - if not SILE.scratch.bibtex.bib[options.key] then + local entry = SILE.scratch.bibtex.bib[options.key] + if not entry then SU.warn("Unknown reference in citation " .. options.key) return end - crossrefResolve(SILE.scratch.bibtex.bib, options.key) + if entry.type == "xdata" then + SU.warn("Skipped citation of @xdata entry " .. options.key) + return + end + crossrefAndXDataResolve(SILE.scratch.bibtex.bib, entry) local style = SILE.settings:get("bibtex.style") local bibstyle = require("packages.bibtex.styles." .. style) local cite, err = Bibliography.produceReference(options, SILE.scratch.bibtex.bib, bibstyle) From 742a0c425787819c2cd14b950d5b7959669c86a6 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sun, 9 Jun 2024 18:41:44 +0200 Subject: [PATCH 11/32] fix(packages): Ignore `@preamble` in bibTeX bibliography Skip `@preamble` as done for `@comment`. The `@preamble` command is used to provide TeX or LaTeX macros that are processed by BibTeX program, but it is unspecified what to do with these when converting to a format that does not use TeX. See #2051 --- packages/bibtex/init.lua | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index 6d3731b6c..5665e7e6c 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -21,16 +21,18 @@ local bibtexparser = epnf.define(function (_ENV) local value = balanced + doubleq + myID local pair = Cg(myTag * _ * "=" * _ * C(value)) * _ * sep^-1 / function (...) local t= {...}; return t[1], t[#t] end local list = Cf(Ct("") * pair^0, rawset) - local commentKey = Cmt(R("az", "AZ")^1, function(_, _, a) - return a:lower() == "comment" + local skippedType = Cmt(R("az", "AZ")^1, function(_, _, tag) + -- ignore both @comment and @preamble + local t = tag:lower() + return t == "comment" or t == "preamble" end) START "document" - document = (V"comment" + V"entry")^1 -- order important: @comment must have precedence over @other + document = (V"skipped" + V"entry")^1 -- order important: skipped (@comment, @preamble) must be first * (-1 + E("Unexpected character at end of input")) - comment = WS + - ( V"blockcomment" + (P"%" * (1-S"\r\n")^0 * S"\r\n") / function () return "" end) -- Don't bother telling me about comments - blockcomment = (P("@") * commentKey) + balanced / function () return "" end -- Don't bother telling me about comments + skipped = WS + + ( V"blockskipped" + (P"%" * (1-S"\r\n")^0 * S"\r\n") / "") + blockskipped = (P("@") * skippedType) + balanced / "" entry = Ct( P("@") * Cg(myTag, "type") * _ * P("{") * _ * Cg(myID, "label") * _ * sep * list * P("}") * _ ) end) -- luacheck: pop From 63083addace0925eb5d83803a1a86465d84cf797 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sun, 9 Jun 2024 20:44:10 +0200 Subject: [PATCH 12/32] feat(packages): Support `@string` syntax in bibTeX bibliography Support `@string` blocks in a bibtex file, and the use of the thus defined strings in other entry fields. Supports concatenation with the "#" operator. As part of the syntax clean-up, all content between entries is now skipped (it doesn't need to start with a percent sign, everything that is not an entry is a comment. See #2051 --- packages/bibtex/init.lua | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index 5665e7e6c..811934776 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -11,14 +11,17 @@ local Bibliography -- stylua: ignore start ---@diagnostic disable: undefined-global, unused-local, lowercase-global local bibtexparser = epnf.define(function (_ENV) + local strings = {} -- Local store for @string entries + local identifier = (SILE.parserBits.identifier + S":-")^1 local balanced = C{ "{" * P" "^0 * C(((1 - S"{}") + V(1))^0) * "}" } / function (...) local t={...}; return t[2] end - local doubleq = C( P'"' * C(((1 - S'"\r\n\f\\') + (P'\\' * 1)) ^ 0) * '"' ) + local quoted = C( P'"' * C(((1 - S'"\r\n\f\\') + (P'\\' * 1)) ^ 0) * '"' ) / function (...) local t={...}; return t[2] end local _ = WS^0 local sep = S",;" * _ - local myID = C(identifier + P(1)) / function (t) return t end + local myID = C(identifier + P(1)) / function (t) return strings[t] or t end local myTag = C(identifier + P(1)) / function (t) return t:lower() end - local value = balanced + doubleq + myID + local pieces = balanced + quoted + myID + local value = Ct(pieces * (WS * P("#") * WS * pieces)^0) / function (t) return table.concat(t) end local pair = Cg(myTag * _ * "=" * _ * C(value)) * _ * sep^-1 / function (...) local t= {...}; return t[1], t[#t] end local list = Cf(Ct("") * pair^0, rawset) local skippedType = Cmt(R("az", "AZ")^1, function(_, _, tag) @@ -28,11 +31,16 @@ local bibtexparser = epnf.define(function (_ENV) end) START "document" - document = (V"skipped" + V"entry")^1 -- order important: skipped (@comment, @preamble) must be first + document = (V"skipped" -- order important: skipped (@comment, @preamble) must be first + + V"stringblock" -- order important: @string must be before @entry + + V"entry")^1 * (-1 + E("Unexpected character at end of input")) - skipped = WS + - ( V"blockskipped" + (P"%" * (1-S"\r\n")^0 * S"\r\n") / "") + skipped = WS + (V"blockskipped" + (1 - P"@")^1 ) / "" blockskipped = (P("@") * skippedType) + balanced / "" + stringblock = Ct( P("@string") * _ * P("{") * pair * _ * P("}") * _ ) + / function (t) + strings[t[1]] = t[2] + return t end entry = Ct( P("@") * Cg(myTag, "type") * _ * P("{") * _ * Cg(myID, "label") * _ * sep * list * P("}") * _ ) end) -- luacheck: pop From 646e3a4ef613d13db9491b87127f01d28f549b23 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Fri, 14 Jun 2024 20:46:05 +0200 Subject: [PATCH 13/32] feat(packages): Biblatex data inheritance and field mapping Add (a part of) the BibLaTeX data inheritance rules for cross-references. Use BibLaTeX field names, but still support the legacy BibTeX file names. As part of these refactors, allow loading more than one bibliography file. --- packages/bibtex/bibmaps.lua | 60 +++++++++++++++++++++++++ packages/bibtex/init.lua | 70 +++++++++++++++++++++++++----- packages/bibtex/styles/chicago.lua | 10 ++--- 3 files changed, 125 insertions(+), 15 deletions(-) create mode 100644 packages/bibtex/bibmaps.lua diff --git a/packages/bibtex/bibmaps.lua b/packages/bibtex/bibmaps.lua new file mode 100644 index 000000000..cdf02aec4 --- /dev/null +++ b/packages/bibtex/bibmaps.lua @@ -0,0 +1,60 @@ +-- Mappings for aliases and inheritance rules + +-- Partial implementation of the Biber/BibLaTeX data inheritance rules +-- (derived from the biblatex package manual v3.20, appendix A) +-- FIXME: This is not complete +local crossrefmap = { + book = { + inbook = { + author = "author", -- inbook inherits author from book author + bookauthor = "author", -- inbook inherits bookauthor from book author + indexsorttitle = false, -- inbook skips (=does not inherit) indexsorttitle from book + indextitle = false, + shorttitle = false, + sorttitle = false, + subtitle = "booksubtitle", + title = "booktitle", + titleaddon = "booktitleaddon", + }, + }, + periodical = { + article = { + indexsorttitle = false, + indextitle = false, + shorttitle = false, + sorttitle = false, + subtitle = "journalsubtitle", + title = "journaltitle", + titleaddon = "journaltitleaddon", + }, + }, + proceedings = { + inproceedings = { + indexsorttitle = false, + indextitle = false, + shorttitle = false, + sorttitle = false, + subtitle = "booksubtitle", + title = "booktitle", + titleaddon = "booktitleaddon", + }, + }, +} + +-- biblatex field aliases +-- From biblatex package manual v3.20, section 2.2.5 +local fieldmap = { + address = "location", + annote = "annotation", + archiveprefix = "eprinttype", + key = "sortkey", + pdf = "file", + journal = "journaltitle", + primaryclass = "eprintclass", + school = "institution", +} + +return { + crossrefmap = crossrefmap, + fieldmap = fieldmap, +} diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index 811934776..47750c2e2 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -47,7 +47,30 @@ end) -- stylua: ignore end ---@diagnostic enable: undefined-global, unused-local, lowercase-global -local parseBibtex = function (fn) +local bibcompat = require("packages.bibtex.bibmaps") +local crossrefmap, fieldmap = bibcompat.crossrefmap, bibcompat.fieldmap + +local function consolidateEntry (entry, label) + local consolidated = {} + for field, value in pairs(entry.attributes) do + consolidated[field] = value + local alias = fieldmap[field] + if alias then + if entry.attributes[alias] then + SU.warn("Duplicate field '" .. field .. "' and alias '" .. alias .. "' in entry '" .. label .. "'") + else + consolidated[alias] = value + end + end + end + entry.attributes = consolidated + return entry +end + +--- Parse a BibTeX file and populate a bibliography table. +-- @tparam string fn Filename +-- @tparam table biblio Table of entries +local function parseBibtex (fn, biblio) fn = SILE.resolveFile(fn) or SU.error("Unable to resolve Bibtex file " .. fn) local fh, e = io.open(fn) if e then @@ -58,14 +81,45 @@ local parseBibtex = function (fn) if not t or not t[1] or t.id ~= "document" then SU.error("Error parsing bibtex") end - local entries = {} for i = 1, #t do if t[i].id == "entry" then local ent = t[i][1] - entries[ent.label] = { type = ent.type, attributes = ent[1] } + local entry = { type = ent.type, attributes = ent[1] } + if biblio[ent.label] then + SU.warn("Duplicate entry key '" .. ent.label .. "', picking the last one") + end + biblio[ent.label] = consolidateEntry(entry, ent.label) + end + end +end + +--- Copy fields from the parent entry to the child entry. +-- BibLaTeX/Biber have a complex inheritance system for fields. +-- This implementation is more naive, but should be sufficient for reasonable +-- use cases. +-- @tparam table parent Parent entry +-- @tparam table entry Child entry +local function fieldsInherit (parent, entry) + local map = crossrefmap[parent.type] and crossrefmap[parent.type][entry.type] + if not map then + -- @xdata and any other unknown types: inherit all missing fields + for field, value in pairs(parent.attributes) do + if not entry.attributes[field] then + entry.attributes[field] = value + end + end + return -- done + end + for field, value in pairs(parent.attributes) do + if map[field] == nil and not entry.attributes[field] then + entry.attributes[field] = value + end + for childfield, parentfield in pairs(map) do + if parentfield and not entry.attributes[parentfield] then + entry.attributes[parentfield] = parent.attributes[childfield] + end end end - return entries end --- Resolve the 'crossref' and 'xdata' fields on a bibliography entry. @@ -104,11 +158,7 @@ local function crossrefAndXDataResolve (bib, entry) local parent = bib[ref] if parent then crossrefAndXDataResolve(bib, parent) - for k, v in pairs(parent.attributes) do - if not entry.attributes[k] then - entry.attributes[k] = v - end - end + fieldsInherit(parent, entry) else SU.warn("Unknown crossref " .. ref .. " in bibliography entry " .. entry.label) end @@ -133,7 +183,7 @@ end function package:registerCommands () self:registerCommand("loadbibliography", function (options, _) local file = SU.required(options, "file", "loadbibliography") - SILE.scratch.bibtex.bib = parseBibtex(file) -- Later we'll do multiple bibliogs, but not now + parseBibtex(file, SILE.scratch.bibtex.bib) end) self:registerCommand("bibstyle", function (_, _) diff --git a/packages/bibtex/styles/chicago.lua b/packages/bibtex/styles/chicago.lua index fe34a7424..3dd11d7af 100644 --- a/packages/bibtex/styles/chicago.lua +++ b/packages/bibtex/styles/chicago.lua @@ -20,7 +20,7 @@ local ChicagoStyles = pl.tablex.merge(Bibliography.Style, { ". ", quotes(title, "."), " ", - italic(journal), + italic(journaltitle), optional(" ", volume), optional(" no. ", number), optional(" ", parens(optional(month, " "), year)), @@ -34,7 +34,7 @@ local ChicagoStyles = pl.tablex.merge(Bibliography.Style, { ". ", quotes(title, "."), " ", - italic(journal), + italic(journaltitle), optional(", ", month), optional(", ", year), optional(": ", pageRange), @@ -59,7 +59,7 @@ local ChicagoStyles = pl.tablex.merge(Bibliography.Style, { " ", optional("In ", italic(booktitle), ". "), optional(transEditor, ". "), - optional(address, ": "), + optional(location, ": "), optional(pub, year and ", " or ". "), optional(year, ". "), optional(number, ". "), @@ -70,7 +70,7 @@ local ChicagoStyles = pl.tablex.merge(Bibliography.Style, { italic(title), ". ", optional(transEditor, ". "), - optional(address, ": "), + optional(location, ": "), optional(pub, year and ", " or ". "), optional(year, ". "), optional(number, ". "), @@ -85,7 +85,7 @@ local ChicagoStyles = pl.tablex.merge(Bibliography.Style, { " ", optional(transEditor, ". "), optional(bibtype, ". "), -- "type" from BibTeX entry - optional(address, ": "), + optional(location, ": "), optional(pub, ", "), optional(year, ".") end, From 7c416b56803c4744ce17647aa9ab685ebba57606 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Fri, 14 Jun 2024 21:25:37 +0200 Subject: [PATCH 14/32] docs(packages): Describe the supported bibtex syntax --- packages/bibtex/init.lua | 86 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index 47750c2e2..f11de2be7 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -252,6 +252,92 @@ To produce a full reference, use \autodoc:command{\reference{}}. Currently, the only supported bibliography style is Chicago referencing, but other styles should be easy to implement. Adapt \code{packages/bibtex/styles/chicago.lua} as necessary. + +\smallskip +\noindent +\em{Notes on the supported BibTeX syntax} +\novbreak + +\indent +The BibTeX file format is a plain text format for bibliographies. + +The \code{@type\{…\}} syntax is used to specify an entry, where \code{type} is the type of the entry, and is case-insensitive. +Any content outside entries is ignored. + +The \code{@preamble} and \code{@comment} special entries are ignored. +The former is specific to TeX-based systems, and the latter is a comment (everything between the balanced braces is ignored). + +The \code{@string\{key=value\}} special entry is used to define a string or “abbreviation,” for use in other subsequent entries. + +The \code{@xdata} entry is used to define an entry that can be used as a reference in other entries. +Such entries are not printed in the bibliography. +Normally, they cannot be cited directly. +In this implementation, a warning is raised if they are; but as they have no known type, their formatting is not well-defined, and might not be meaningful. + +Regular bibliography entries have the following syntax: + +\begin[type=autodoc:codeblock]{raw} +@type{key, + field1 = value1, + field2 = value2, + … +} +\end{raw} + +The entry key is a unique identifier for the entry, and is case-sensitive. +Entries consist of fields, which are key-value pairs. +The field names are case-insensitive. +Spaces and line breaks are not important, except for readability. +On the contrary, commas are compulsory between any two fields of an entry. + +String values shall be enclosed in either double quotes or curly braces. +The latter allows using quotes inside the string, while the former does not without escaping them with a backslash. + +When string values are not enclosed in quotes or braces, they must not contain any whitespace characters. +The value is then considered to be a refererence to an abbreviation previously defined in a \code{@string} entry. +If no such abbreviation is found, the value is considered to be a string literal. +(This allows a decent fallback for fields where curly braces or double quotes could historically be omitted, such as numerical values, and one-word strings.) + +String values are assumed to be in the UTF-8 encoding, and shall not contain (La)TeX commands. +Special character sequences from TeX (such as \code{`} assumed to be an opening quote) are not supported. + +Values can also be composed by concatenating strings, using the \code{#} character. + +Besides using string references, entries have two other \em{parent-child} inheritance mechanisms allowing to reuse fields from other entries, without repeating them: the \code{crossref} and \code{xdata} fields. + +The \code{crossref} field is used to reference another entry by its key. +The \code{xdata} field accepts a comma-separated list of keys of entries that are to be inherited. + +Some BibTeX implementations automatically include entries referenced with the \code{crossref} field in the bibliography, when a certain threshold is met. +This implementation does not do that. + +Depending on the types of the parent and child entries, the child entry may inherit some or all fields from the parent entry, and some inherited fields may be reassigned in the child entry. +For instance, the \code{title} in a \code{@collection} entry is inherited as the \code{booktitle} field in a \code{@incollection} child entry. +Some BibTeX implementations allow configuring the data inheritance behavior, but this implementation does not. +It is also currently quite limited on the fields that are reassigned, and only provides a subset of the mappings defined in the BibLaTeX manual, appendix B. + +Here is an example of a BibTeX file showing some of the abovementioned features: + +\begin[type=autodoc:codeblock]{raw} +@string{JIT = "Journal of Interesting Things"} +... +This text is ignored +... +@xdata{jit-vol1-iss2, + journal = JIT # { (JIT)}, + year = {2020}, + month = {jan}, + volume = {1}, + number = {2}, +} +@article{my-article, + author = {Doe, John and Smith, Jane} + title = {Theories & Practices}, + xdata = {jit-1-2}, + pages = {100--200}, +} +\end{raw} + \end{document} ]] From b775147d4b50fad2c116553d1312d0edad929711 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sat, 15 Jun 2024 12:00:52 +0200 Subject: [PATCH 15/32] chore(packages): Fixup formatting of an error message --- packages/bibtex/bibliography.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bibtex/bibliography.lua b/packages/bibtex/bibliography.lua index 4a0fc646e..1821f9f0a 100644 --- a/packages/bibtex/bibliography.lua +++ b/packages/bibtex/bibliography.lua @@ -200,7 +200,7 @@ do von_start, last_lim, jr_lim, first_start, first_lim = 1, commas[1], commas[2], commas[2], n + 1 divide_von_from_last() else - SU.error("Too many commas in name '%s'") + SU.error(("Too many commas in name '%s'"):format(str)) end -- = -- We set long and short forms together; [[ss]] is the From aaa613f636e6c892c70ea560a0815999ee617b2e Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sat, 15 Jun 2024 12:08:41 +0200 Subject: [PATCH 16/32] fix(packages): Correct handling of ampersands and tildes in bibtex Accept `\&` for compatibility with legacy BibTeX, but do not mandate it to be escaped for compatibility with other engines. Support unescaped `~` as a non-breaking space for compability with TeX, this is often found in existing bibliography files. Support `\~` to render a tilde. XML-escape the input so it can safely be wrapped in a `` construct. Closes #2050 Closes #1860 (replaced by this implementation) --- packages/bibtex/init.lua | 45 +++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/packages/bibtex/init.lua b/packages/bibtex/init.lua index f11de2be7..c4b1d7561 100644 --- a/packages/bibtex/init.lua +++ b/packages/bibtex/init.lua @@ -7,6 +7,37 @@ local epnf = require("epnf") local Bibliography +local nbsp = luautf8.char(0x00A0) +local function sanitize (str) + local s = str + -- TeX special characters: + -- Backslash-escaped tilde is a tilde, + -- but standalone tilde is a non-breaking space + :gsub( + "(.?)~", + function (prev) + if prev == "\\" then + return "~" + end + return prev .. nbsp + end + ) + -- Other backslash-escaped characters are skipped + -- TODO FIXME: + -- This ok for \", \& etc. which we want to unescape, + -- BUT what should we do with other TeX-like commands? + :gsub( + "\\", + "" + ) + -- We will wrap the content in tags so we need to XML-escape + -- the input. + :gsub("&", "&") + :gsub("<", "<") + :gsub(">", ">") + return s +end + -- luacheck: push ignore -- stylua: ignore start ---@diagnostic disable: undefined-global, unused-local, lowercase-global @@ -18,11 +49,14 @@ local bibtexparser = epnf.define(function (_ENV) local quoted = C( P'"' * C(((1 - S'"\r\n\f\\') + (P'\\' * 1)) ^ 0) * '"' ) / function (...) local t={...}; return t[2] end local _ = WS^0 local sep = S",;" * _ - local myID = C(identifier + P(1)) / function (t) return strings[t] or t end - local myTag = C(identifier + P(1)) / function (t) return t:lower() end - local pieces = balanced + quoted + myID - local value = Ct(pieces * (WS * P("#") * WS * pieces)^0) / function (t) return table.concat(t) end - local pair = Cg(myTag * _ * "=" * _ * C(value)) * _ * sep^-1 / function (...) local t= {...}; return t[1], t[#t] end + local myID = C(identifier) + local myStrID = myID / function (t) return strings[t] or t end + local myTag = C(identifier) / function (t) return t:lower() end + local pieces = balanced + quoted + myStrID + local value = Ct(pieces * (WS * P("#") * WS * pieces)^0) + / function (t) return table.concat(t) end / sanitize + local pair = myTag * _ * "=" * _ * value * _ * sep^-1 + / function (...) local t= {...}; return t[1], t[#t] end local list = Cf(Ct("") * pair^0, rawset) local skippedType = Cmt(R("az", "AZ")^1, function(_, _, tag) -- ignore both @comment and @preamble @@ -300,6 +334,7 @@ If no such abbreviation is found, the value is considered to be a string literal String values are assumed to be in the UTF-8 encoding, and shall not contain (La)TeX commands. Special character sequences from TeX (such as \code{`} assumed to be an opening quote) are not supported. +There are exceptions to this rule. Notably, the \code{~} character can be used to represent a non-breaking space (when not backslash-escaped), and the \code{\\&} sequence is accepted (though this implementation does not mandate escaping ampersands). Values can also be composed by concatenating strings, using the \code{#} character. From d59535e3a51e149bf11d88da96b2bc170a8978a2 Mon Sep 17 00:00:00 2001 From: Omikhleia Date: Sat, 15 Jun 2024 15:02:22 +0200 Subject: [PATCH 17/32] test(packages): Add more syntax elements to bibtex test Test `@preamble`, `@xdata`, `@string`, string concatenation with hash `#`, `xdata` and `crossref` fields. This test file is already insanely long, so rather than adding new tests, I preferred hacking a few existing entries: The result should thus meet the same expectation although the bibtex file encodes the information slightly differently. --- tests/biblio-tests.bib | 73 +++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/tests/biblio-tests.bib b/tests/biblio-tests.bib index 170916399..dd2c71faa 100644 --- a/tests/biblio-tests.bib +++ b/tests/biblio-tests.bib @@ -1,4 +1,18 @@ -% -- article +-- SILE skips these fields: +@preamble{"\newcommand{\noop}[1]{}"} +@comment{jabref-meta: databaseType:bibtex;} + +-- strings and concatenation +@string{journalof = {The Journal of }} +@string{myjournal = journalof # {Typography}} + +-- xdata +% Not supposed to be referenced, but supplementing other entries. +@xdata{sileorg, + publisher = "SILE " # "organization", +} + +-- article % % An article from a journal or magazine. % Required fields: author, title, journal, year. @@ -7,11 +21,11 @@ @article{MinimalArticle, author = "Doe, John", title = "My Awesome Article", - journal = "The Journal of Typography", + journaltitle = myjournal, year = 2022, } -% With month remove, as a check for the chigago style +% With month removed, as a check for the chigago style @article{FullArticle, author = "Doe, John", title = "My Complete Article", @@ -34,7 +48,7 @@ @article{NewspaperArticle url = "https://technobabblers.com/2020/123" } -% -- book +-- book % % A book with an explicit publisher. % Required fields: author or editor, title, publisher, year. @@ -43,14 +57,14 @@ @article{NewspaperArticle @book{MinimalBook1, author = "Doe, John and Smith, Jane", title = "A Minimal Book Entry", - publisher = "SILE organization", + xdata = "sileorg", year = 2020, } @book{MinimalBook2, editor = "Doe, John and Smith, Jane", title = "Another Minimal Book Entry", - publisher = "SILE organization", + xdata = "sileorg", address = "London", year = 2022, } @@ -58,8 +72,7 @@ @book{MinimalBook2 @book{FullBook, author = "Doe, John and Smith, Jane", title = "A Complete Book", - publisher = "SILE organization", - address = "London", + xdata = "sileorg", year = 2022, volume = 10, series = "The Collected Templates", @@ -69,7 +82,7 @@ @book{FullBook note = "illustrated", } -% -- booklet +-- booklet % % A work that is printed and bound, but without a named publisher or sponsoring institution. % Required field: title. @@ -89,7 +102,7 @@ @booklet{FullBooklet note = "Accessed: 2022" } -% -- conference +-- conference % % = Same as "inproceedings" @@ -110,7 +123,7 @@ @conference{FullConference note = "Incl. 5 figures", } -% -- inbook +-- inbook % % A part of a book, which may be a chapter (or section or whatever) and/or a range of pages. % Required fields: author or editor, title, chapter and/or pages, publisher, year. @@ -141,7 +154,7 @@ @inbook{FullInBook note = "Revised edition" } -% -- incollection +-- incollection % % A part of a book having its own title. % Required fields: author, title, booktitle, publisher, year. @@ -156,11 +169,7 @@ @incollection{MinimalInCollection } @incollection{FullInCollection, - author = "Doe, John and Smith, Jane", - title = "Our In-Collection Contribution", - booktitle = "The Book of Everyting", - publisher = "Springer", - year = 2022, + crossref = {MinimalInCollection}, publisher = "SILE organization", editor = "Wills, Donald", volume = 10, @@ -173,7 +182,7 @@ @incollection{FullInCollection note = "revised and augmented" } -% -- inproceedings +-- inproceedings % % An article in a conference proceedings. % Required fields: author, title, booktitle, year. @@ -196,7 +205,7 @@ @inproceedings{FullInProceedings note = "Incl. 5 figures", } -% -- manual +-- manual % % Technical documentation. % Required field: title. @@ -217,7 +226,7 @@ @manual{FullManual note = "unabridged" } -% -- mastersthesis +-- mastersthesis % % A Master’s thesis. % Required fields: author, title, school, year. @@ -241,7 +250,7 @@ @mastersthesis{FullMastersThesis note = "University archives" } -% -- misc +-- misc % % Use this type when nothing else fits... % Required fields: none. WE WILL ASSUME AT LEAST A TITLE! @@ -256,7 +265,7 @@ @misc{FullMisc note = "Last accessed: 2020-05-06" } -% -- phdthesis +-- phdthesis % % A PhD thesis % Required fields: author, title, school, year. @@ -274,7 +283,7 @@ @phdthesis{FullPhdThesis note = "University archives" } -% -- proceedings +-- proceedings % % The proceedings of a conference. % Required fields: title, year. @@ -290,7 +299,7 @@ @proceedings{FullProceedings year = 2018 } -% -- techreport +-- techreport % % A report published by a school or other institution, usually numbered within a series. % Required fields: author, title, institution, year. @@ -308,7 +317,7 @@ @techreport{FullTechReport note = "Technical note" } -% -- unpublished +-- unpublished % % A document having an author and title, but not formally published. % Required fields: author, title, note. @@ -328,14 +337,12 @@ @unpublished{FullUnpublished year = 2023, } -@comment{jabref-meta: databaseType:bibtex;} - -% -- comment +-- comment % -% Should not be parsed -% For test: same "MinimalArticle' key as above, to check this is not used! +% Should not be parsed, as stated above. +% For test: same "MinimalArticle' key as above, to check this is indeed not used! @comment{MinimalArticle, - Author = {Doe, John}, - Title = {I SHALL NOT BE VISIBLE}, - Year = {2022}, + author = {Doe, John}, + title = {I SHALL NOT BE VISIBLE}, + year = {2022}, } From 3724f85d2611f8b3cbd61da2d1065115942d3da1 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 26 Jun 2024 12:37:01 +0300 Subject: [PATCH 18/32] refactor(tooling): Cleanup operation order and comments in autoconf setup --- configure.ac | 73 ++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/configure.ac b/configure.ac index 4107e4cf6..23d1af5b7 100644 --- a/configure.ac +++ b/configure.ac @@ -5,22 +5,26 @@ AC_CONFIG_MACRO_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign tar-pax dist-zstd dist-zip no-dist-gzip color-tests subdir-objects]) AM_SILENT_RULES([yes]) +# Load up re-usable m4 bits shared across Caleb's projects QUE_GIT_VERSION QUE_TRANSFORM_PACKAGE_NAME QUE_DEVELOPER_MODE QUE_DIST_CHECKSUMS -SILE_DIST_CHECKSUMS -# Checks for programs. -AC_PROG_CC -AC_PROG_OBJC +# Extend the QUE checksums feature with support for the PDF manual +SILE_DIST_CHECKSUMS +# Check for and set variables for tooling used during our build process AC_PROG_AWK +AC_PROG_CC AC_PROG_GREP +AC_PROG_OBJC AC_PROG_SED - QUE_PROGVAR([cmp]) QUE_PROGVAR([find]) +QUE_PROGVAR([jq]) +QUE_PROGVAR([pdfinfo]) +QUE_PROGVAR([xargs]) LT_PREREQ([2.2]) LT_INIT([dlopen]) @@ -89,13 +93,6 @@ AM_CONDITIONAL([MANUAL], [test "x$with_manual" = "xyes"]) AM_CONDITIONAL([FONT_DOWNLOAD_TOOLS], [test -z ${DEVELOPER_MODE_TRUE} || (test -z ${SOURCE_IS_DIST_TRUE} && test -z ${MANUAL_TRUE})]) AC_SUBST([FONT_DOWNLOAD_TOOLS]) -AC_MSG_NOTICE([checking for SILE specific build dependencies]) - -QUE_FONT(Gentium Plus) - -QUE_PROGVAR([pdfinfo]) -QUE_PROGVAR([jq]) -QUE_PROGVAR([xargs]) AM_COND_IF([MANUAL], [ QUE_PROGVAR([dot]) QUE_PROGVAR([gs]) @@ -125,6 +122,7 @@ case $host_os in AC_MSG_RESULT([no]) ;; esac +AM_CONDITIONAL([APPKIT], [test "x$have_appkit" = "xyes"]) AM_COND_IF([FONTCONFIG], [PKG_CHECK_MODULES(FONTCONFIG, [fontconfig], @@ -161,6 +159,31 @@ PKG_CHECK_MODULES(ICU, icu-uc icu-io, [ AC_MSG_FAILURE([Required ICU library not found]) fi ]) +AM_CONDITIONAL([ICU], [test "x$with_icu" = "xyes"]) + +# Required for downloading fonts for the manual and for tests +# Since the source tarball includes a prebuilt manual we only need this for Git source builds +AM_COND_IF([FONT_DOWNLOAD_TOOLS], [ + QUE_PROGVAR([curl]) + QUE_PROGVAR([bsdtar]) +]) + +AM_COND_IF([DEVELOPER_MODE], [ + QUE_PROGVAR([busted]) + QUE_PROGVAR([curl]) + QUE_PROGVAR([delta]) + QUE_PROGVAR([diff]) + QUE_PROGVAR([head]) + QUE_PROGVAR([ldoc]) + QUE_PROGVAR([luacheck]) + QUE_PROGVAR([luarocks]) + QUE_PROGVAR([nix]) + QUE_PROGVAR([npm]) + QUE_PROGVAR([perl]) + QUE_PROGVAR([sort]) + QUE_PROGVAR([stylua]) + QUE_PROGVAR([tr]) +]) AX_PROG_LUA([5.1]) AX_LUA_HEADERS @@ -196,32 +219,8 @@ AM_COND_IF([SYSTEM_LUAROCKS], [ QUE_PROGVAR([luarocks]) ]) -# Required for downloading fonts for the manual and for tests -# Since the source tarball includes a prebuilt manual we only need this for Git source builds -AM_COND_IF([FONT_DOWNLOAD_TOOLS], [ - QUE_PROGVAR([curl]) - QUE_PROGVAR([bsdtar]) -]) - -AM_COND_IF([DEVELOPER_MODE], [ - QUE_PROGVAR([busted]) - QUE_PROGVAR([curl]) - QUE_PROGVAR([delta]) - QUE_PROGVAR([diff]) - QUE_PROGVAR([head]) - QUE_PROGVAR([ldoc]) - QUE_PROGVAR([luacheck]) - QUE_PROGVAR([luarocks]) - QUE_PROGVAR([nix]) - QUE_PROGVAR([npm]) - QUE_PROGVAR([perl]) - QUE_PROGVAR([sort]) - QUE_PROGVAR([stylua]) - QUE_PROGVAR([tr]) -]) +QUE_FONT(Gentium Plus) -AM_CONDITIONAL([APPKIT], [test "x$have_appkit" = "xyes"]) -AM_CONDITIONAL([ICU], [test "x$with_icu" = "xyes"]) AC_SUBST([APPKIT_TRUE]) AC_SUBST([FONTCONFIG_TRUE]) From e277d48f40fc689aea693e834ef059ec36e165c2 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 27 Jun 2024 16:27:16 +0300 Subject: [PATCH 19/32] docs(api): Fix typos and copy-edit some doc strings --- core/globals.lua | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/globals.lua b/core/globals.lua index d156e70af..92981eaca 100644 --- a/core/globals.lua +++ b/core/globals.lua @@ -2,24 +2,24 @@ -- @module globals -- @alias _G ---- Penlight od-demand loader. +--- Penlight on-demand loader. -- The Lua language adopts a "no batteries included" philosophy by providing a minimal standard library. Penlight is --- a widely used set libraries for making it easier to work with common tasks. Loading SILE implies that the PEnlight +-- a widely used set libraries for making it easier to work with common tasks. Loading SILE implies that the Penlight -- on-demand module loader is available, allowing any Penlight functions to be accessed using the `pl` prefix. Consult -- the [Penlight documentation](https://lunarmodules.github.io/Penlight/) for specifics of the utilities available. _G.pl = require("pl.import_into")() --- UTF-8 string library. --- LuaJIT 5.1 and 5.2's `string` module only handle strings as bytes. Lua 5.3+ has a UTF-8 safe `string` module, but its --- feature set is somewhat underwhelming. This module includes more functions and levels the playing field no matter --- which Lua VM is being used. See [luautf8 docs](https://github.com/starwing/luautf8) for more details. +-- The standard `string` modules in LuaJIT as well as PUC Lua 5.1 and 5.2 only handle strings as bytes. Lua 5.3+ has a +-- UTF-8 safe `string` module but its feature set is underwhelming. This adds more functions and levels the playing +-- field no matter which Lua VM is being used. See [luautf8 docs](https://github.com/starwing/luautf8) for details. _G.luautf8 = require("lua-utf8") --- Fluent localization library. --- For handling messages in various languages SILE provides an implementation of [Project --- Fluent](https://projectfluent.org/)'s localization system (originally developed by Mozilla for use in Firefox). This --- global is an instantiated interface to [fluent-lua](https://github.com/alerque/fluent-lua) pre-loaded with resources --- for all the langugaes and regions SILE has support for. +-- For handling messages in various languages SILE provides an implementation of +-- [Project Fluent](https://projectfluent.org/)'s localization system (originally developed by Mozilla for use in +-- Firefox). This global is an instantiated interface to [fluent-lua](https://github.com/alerque/fluent-lua) pre-loaded +-- with resources for all the langugaes and regions SILE has support for. _G.fluent = require("fluent")() -- For developer testing only, usually in CI From f9506d116d6bc9fea0fbd309d8474441e714e337 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 27 Jun 2024 16:41:00 +0300 Subject: [PATCH 20/32] style(cli): Avoid raw strings where not needed --- src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ae6a911ed..801344833 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,8 +59,8 @@ pub fn inject_version(lua: &Lua) { pub fn load_sile(lua: &Lua) { let entry: LuaString = lua.create_string("core.sile").unwrap(); - let r#require: LuaFunction = lua.globals().get("require").unwrap(); - r#require.call::(entry).unwrap(); + let require: LuaFunction = lua.globals().get("require").unwrap(); + require.call::(entry).unwrap(); } pub fn version() -> crate::Result { @@ -186,8 +186,8 @@ pub fn run( finish.call::<_, _>(())?; } else { let repl_module: LuaString = lua.create_string("core.repl")?; - let r#require: LuaFunction = lua.globals().get("require")?; - let repl: LuaTable = r#require.call::(repl_module)?; + let require: LuaFunction = lua.globals().get("require")?; + let repl: LuaTable = require.call::(repl_module)?; repl.call_method::<_, _>("enter", ())?; } Ok(()) From 26a63463346a73eec38062984a8fec75761238b7 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 26 Jun 2024 14:20:35 +0300 Subject: [PATCH 21/32] fix(build): Do not distribute main man page, requires Rust tooling Requiring us to download Rust dependencies and build the whole Rust CLI just to generate the source dist tarball is a waste. Distributing the man page in the distribution package makes sense when generating it requires tooling the user may not have, but in this case it *will* get rebuild when they generate the CLI anyway. --- Makefile.am | 1 + build-aux/que_rust_boilerplate.am | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 7ea15a56b..9004d430e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,6 +59,7 @@ BUILT_LUA_SOURCES = core/features.lua core/pathsetup.lua core/version.lua bin_PROGRAMS = sile bin_SCRIPTS = sile-lua +nodist_man_MANS = dist_man_MANS = sile-lua.1 sile_SOURCES = src/bin/sile.rs src/lib.rs src/cli.rs EXTRA_sile_SOURCES = diff --git a/build-aux/que_rust_boilerplate.am b/build-aux/que_rust_boilerplate.am index 2f7b2dd48..93514b945 100644 --- a/build-aux/que_rust_boilerplate.am +++ b/build-aux/que_rust_boilerplate.am @@ -2,7 +2,7 @@ export VERSION_FROM_AUTOTOOLS = v$(VERSION) @PACKAGE_VAR@_SOURCES += Cargo.toml build-aux/build.rs EXTRA_@PACKAGE_VAR@_SOURCES += Cargo.lock .version -dist_man_MANS += @PACKAGE_NAME@.1 +nodist_man_MANS += @PACKAGE_NAME@.1 CLEANFILES += $(bin_PROGRAMS) $(BUILT_SOURCES) $(dist_man_MANS) From bbde8e663dcd65262104c9bc32a10e28c3166e43 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 26 Jun 2024 16:00:45 +0300 Subject: [PATCH 22/32] fix(build): Avoid building sile binaries if not using them --- Makefile.am | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9004d430e..5b1db4190 100644 --- a/Makefile.am +++ b/Makefile.am @@ -44,13 +44,11 @@ TESTPREVIEWS ?= $(addsuffix .pdf,$(basename $(filter-out $(_DISABLEDSRCS),$(_TES include $(wildcard Makefile-distfiles) FIGURES = documentation/fig-input-to-output.pdf - MANUAL := documentation/sile.pdf SILELUA := $(PACKAGE_NAME)-lua if MANUAL _MANUAL = $(MANUAL) - endif $(MANUAL): $(FIGURES) @@ -76,7 +74,7 @@ EXTRA_DIST += build-aux/action-updater.js build-aux/cargo-updater.js build-aux/c EXTRA_DIST += Dockerfile build-aux/docker-bootstrap.sh build-aux/docker-fontconfig.conf hooks/build EXTRA_DIST += default.nix flake.nix flake.lock shell.nix build-aux/pkg.nix EXTRA_DIST += package.json # imported by both Nix and Docker -EXTRA_DIST += $(MANUAL) $(FIGURES) +EXTRA_DIST += $(FIGURES) EXTRA_DIST += src/embed.rs.in EXTRA_DIST += sil.abnf @@ -166,7 +164,7 @@ LOCALPATHS := SILE_PATH="$(PWD);libtexpdf/.libs;justenough/.libs" SILEFLAGS ?= -m $(DEPDIR)/$(basename $@).d -d versions -f fontconfig DRAFT ?= false -dist-hook: $(MANUAL) dist-hook-distfiles +dist-hook: dist-hook-distfiles .PHONY: dist-hook-distfiles dist-hook-distfiles: From f3c3b28b8ed67dc1eb6058de3bef2d7832754984 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 27 Jun 2024 11:52:46 +0300 Subject: [PATCH 23/32] refactor(build): Split phony targets by purpose --- Makefile.am | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 5b1db4190..48f88273f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -164,13 +164,20 @@ LOCALPATHS := SILE_PATH="$(PWD);libtexpdf/.libs;justenough/.libs" SILEFLAGS ?= -m $(DEPDIR)/$(basename $@).d -d versions -f fontconfig DRAFT ?= false -dist-hook: dist-hook-distfiles +dist-hook: dist-hook-decore-automake dist-hook-devendor-luarocks -.PHONY: dist-hook-distfiles -dist-hook-distfiles: +.PHONY: dist-hook-decore-automake +dist-hook-decore-automake: cd $(distdir) - $(SED) -i -e '/^LUAMODULES =/s/=.*/=/' Makefile-distfiles $(top_srcdir)/build-aux/decore-automake.sh + +# Don't take it for granted that the user will use --without-system-luarocks +# when building from source tarballs even though the dist tarball may be +# generated with that configuration. +.PHONY: dist-hook-devendor-luarocks +dist-hook-devendor-luarocks: dist-hook-decore-automake + cd $(distdir) + $(SED) -i -e '/^LUAMODULES =/s/=.*/=/' Makefile-distfiles $(SED) -i -e '/^LUAMODULES/d;/^\tlua_modules/d' Makefile.in # Whether to force tests to run from scratch From 7bdaf16564d96a0755ce112f448ee9eaeeaf7059 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 26 Jun 2024 13:08:03 +0300 Subject: [PATCH 24/32] feat(build): Add a developer target to test build dist without automake --- Makefile.am | 12 +++++++++++- hooks/build | 2 ++ tests/Dockerfile-dist | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/Dockerfile-dist diff --git a/Makefile.am b/Makefile.am index 48f88273f..911f506dc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -204,6 +204,11 @@ selfcheck: | $(bin_PROGRAMS) $(_BUILT_SUBDIRS) .PHONY: docs docs: $(_MANUAL) lua-api-docs +.PHONY: fake-manual +fake-manual: + touch $(FIGURES) + touch $(MANUAL) + .PHONY: lua-api-docs lua-api-docs: lua-api-docs/index.html @@ -287,7 +292,7 @@ patterndeps = $(_FORCED) $(_TEST_DEPS) $(_DOCS_DEPS) | $(bin_PROGRAMS) $(DEPDIRS force: ; PHONY_DEVELOPER_TARGETS = busted compare coverage \ - docker-dep-check docker-ghcr-to-hub gource.webm lint luacheck luarocks-lint \ + docker-dep-check docker-ghcr-to-hub docker-test-dist gource.webm lint luacheck luarocks-lint \ prerelease regression_previews regressions release release-preview stylua tagrelease \ test update_expecteds update_libtexpdf .PHONY: $(PHONY_DEVELOPER_TARGETS) @@ -430,6 +435,11 @@ docker-ghcr-to-hub: $(DOCKER) tag $(GHCR_REGISTRY)/$(GHCR_REPO):$(DOCKER_TAG) $(DOCKER_REGISTRY)/$(DOCKER_REPO):$(DOCKER_TAG) $(docker_push) +docker-test-dist: export DOCKERFILE = tests/Dockerfile-dist +docker-test-dist: export DOCKER_TAG = dist-test +docker-test-dist: + ./hooks/build $(VERSION) + gource.webm: $(MKDIR_P) /tmp/gravatars magick documentation/sile-logo.pdf[0] -density 300 -colorspace RGB -negate -resize 50% /tmp/sile-logo.jpg diff --git a/hooks/build b/hooks/build index 065a6024b..f982069db 100755 --- a/hooks/build +++ b/hooks/build @@ -2,6 +2,7 @@ set -e : "${ARCHTAG:=}" +: "${DOCKERFILE:=Dockerfile}" REVISION=$(git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g') @@ -19,4 +20,5 @@ docker build \ --build-arg RUNTIME_DEPS="$RUNTIME_DEPS" \ --build-arg BUILD_DEPS="$BUILD_DEPS" \ --tag $DOCKER_REPO:$DOCKER_TAG \ + --file ${DOCKERFILE} \ ./ diff --git a/tests/Dockerfile-dist b/tests/Dockerfile-dist new file mode 100644 index 000000000..fadb1d234 --- /dev/null +++ b/tests/Dockerfile-dist @@ -0,0 +1,34 @@ +#syntax=docker/dockerfile:1.2 + +ARG ARCHTAG + +FROM docker.io/library/archlinux:base-devel$ARCHTAG + +ARG RUNTIME_DEPS +ARG BUILD_DEPS + +# Freshen all base system packages +RUN pacman-key --init +RUN pacman --needed --noconfirm -Syq archlinux-keyring +RUN pacman --needed --noconfirm -Suq + +# Install build and run-time dependecies +RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS $BUILD_DEPS + +# Remove autotools, building the source distributions should not require them +RUN pacman --noconfirm -Rn autoconf automake libtool base-devel + +# Set at build time, forces Docker’s layer caching to reset at this point +ARG VERSION + +COPY ./sile-$VERSION /src +WORKDIR /src + +RUN ./configure --with-system-lua-sources --without-manual +RUN make +RUN make install + +# We can't properly test path handling from inside the source directory! +WORKDIR /tmp + +RUN sile --version | grep $VERSION From 2e9705719b5ce3f4fb7d92dd85acaf33fb21bb07 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Wed, 26 Jun 2024 12:53:42 +0300 Subject: [PATCH 25/32] ci(actions): Add workflow to test build source tarball without autotools --- .github/workflows/build.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a21db5bc9..d59540396 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,6 +91,43 @@ jobs: sile-${{ env.CARCH }} sile-${{ env.VERSION }}.sha256.txt + build-from-dist: + runs-on: ubuntu-22.04 + name: Build from source tarball without automake + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Cache lua_modules + uses: actions/cache@v4 + with: + path: | + lua_modules + key: luarocks-${{ hashFiles('Makefile-luarocks', 'sile.rockspec.in') }} + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install jq libarchive-tools libfontconfig1-dev libharfbuzz-dev libicu-dev libluajit-5.1-dev libpng-dev luajit lua-sec lua-socket lua-zlib-dev luarocks + - name: Configure + run: | + ./bootstrap.sh + ./configure \ + --enable-developer-mode \ + BUSTED=false DELTA=false LDOC=false LUACHECK=false NIX=false STYLUA=false FCMATCH=true PDFINFO=false CARGO=true \ + --disable-font-variations \ + --with-system-lua-sources \ + --without-manual + echo "VERSION=$(./build-aux/git-version-gen .tarball-version)" >> $GITHUB_ENV + echo "MAKEFLAGS=-j$(nproc) -Otarget" >> $GITHUB_ENV + - name: Package + run: | + make fake-manual + make distdir + - name: Build from package + run: | + make docker-test-dist + build-nix: runs-on: ubuntu-22.04 name: Build Nix From f634a606b8d6973e2815d7f769b0205d27ba0e3b Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 27 Jun 2024 02:03:38 +0300 Subject: [PATCH 26/32] fix(build): Avoid the perceived need for an extra automake cycle in dist tarball --- configure.ac | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure.ac b/configure.ac index 23d1af5b7..15093c31c 100644 --- a/configure.ac +++ b/configure.ac @@ -290,3 +290,11 @@ AC_OUTPUT AC_MSG_NOTICE([monkey-patching configure to ignore core directory]) m4_esyscmd_s([build-aux/decore-automake.sh]) + +# Avoid the perceived need to run automake again when building from source +# tarballs. The assorted AM include macros assume two passes are needed to +# generate a makefile since one pass has to generate the includes and the +# second one inlines them, but we're distributing a ready-to-use version and +# hence one pass is enough. We can skip ahead and assume the Makefile template +# is ready to go an this point, avoiding the need for autotools at all. +AM_COND_IF([SOURCE_IS_DIST], [touch -r aminclude.am Makefile.in]) From b9d95fdac11ed3e783520dd2b66690db52bad299 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 27 Jun 2024 15:48:31 +0300 Subject: [PATCH 27/32] fix(build): Don't distribute distfiles list with configure option specific output This was preventing --with[out]-system-luarocks from properly figuring out what to install when used from source distributions because it thought it knew already. --- Makefile.am | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 911f506dc..51eed52d2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -40,7 +40,7 @@ TESTPREVIEWS ?= $(addsuffix .pdf,$(basename $(filter-out $(_DISABLEDSRCS),$(_TES # using $(wildcard ...) to include it (the very function we couldn't use because # of the race condition it creates) we actually keep it from being flattened and # hence evaluated when we want it to be. Since the file always exists (see -# BUILT_SOURCES and EXTRA_DIST) this doesn't induce a race. +# BUILT_SOURCES) this doesn't induce a race. include $(wildcard Makefile-distfiles) FIGURES = documentation/fig-input-to-output.pdf @@ -69,7 +69,6 @@ dist_doc_DATA = README.md CHANGELOG.md dist_pdf_DATA = $(_MANUAL) dist_license_DATA = LICENSE.md EXTRA_DIST = spec tests documentation sile-dev-1.rockspec fontconfig.conf -EXTRA_DIST += Makefile-distfiles EXTRA_DIST += build-aux/action-updater.js build-aux/cargo-updater.js build-aux/config.ld build-aux/decore-automake.sh build-aux/git-version-gen build-aux/list-dist-files.sh EXTRA_DIST += Dockerfile build-aux/docker-bootstrap.sh build-aux/docker-fontconfig.conf hooks/build EXTRA_DIST += default.nix flake.nix flake.lock shell.nix build-aux/pkg.nix @@ -177,7 +176,6 @@ dist-hook-decore-automake: .PHONY: dist-hook-devendor-luarocks dist-hook-devendor-luarocks: dist-hook-decore-automake cd $(distdir) - $(SED) -i -e '/^LUAMODULES =/s/=.*/=/' Makefile-distfiles $(SED) -i -e '/^LUAMODULES/d;/^\tlua_modules/d' Makefile.in # Whether to force tests to run from scratch From ea2d501b65f1980c8b88b47dc9eedf6358f24453 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Thu, 27 Jun 2024 18:15:18 +0300 Subject: [PATCH 28/32] fix(build): Flag transitive dependency on Git if *not* using system LuaRocks The lua zlib rock requires Git to fetch sources, hence we need it for vendoring. Sadly the error from LuaRocks isn't getting propogated resulting in an apparently successfull but incomplete install. --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 15093c31c..bb8db44f7 100644 --- a/configure.ac +++ b/configure.ac @@ -217,6 +217,7 @@ AM_COND_IF([SYSTEM_LUAROCKS], [ AX_LUA_MODULE([vstruct], [vstruct]) ], [ QUE_PROGVAR([luarocks]) + QUE_PROGVAR([git]) # required for luarocks to install zlib rock ]) QUE_FONT(Gentium Plus) From f217123d3644c3f7805becc7721414b3d63ef9c6 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 28 Jun 2024 01:55:31 +0300 Subject: [PATCH 29/32] chore(nix): Avoid Nix Flake actually getting treated like the tarball build --- build-aux/pkg.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build-aux/pkg.nix b/build-aux/pkg.nix index 1514da569..1d1e1292d 100644 --- a/build-aux/pkg.nix +++ b/build-aux/pkg.nix @@ -66,8 +66,10 @@ in stdenv.mkDerivation (finalAttrs: { # before, in Nixpkgs, we won't need to run these commands. rm -rf ./libtexpdf cp --no-preserve=mode -r ${libtexpdf-src} ./libtexpdf/ - # pretend to be a tarball release so sile --version will not say `vUNKNOWN`. - echo ${finalAttrs.version} > .tarball-version + # pretend to be a tarball release so sile --version will not say `vUNKNOWN`, + # but don't pretend so hard we make the build system treat us like the tarball. + echo ${finalAttrs.version} > .flake-version + sed -i -e 's/tarball-version/flake-version/' configure.ac ''; nativeBuildInputs = [ From ff9d40fb7a11e1016f6e7ec65aabf7380434f66a Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 28 Jun 2024 11:18:05 +0300 Subject: [PATCH 30/32] chore(deps): Bump pinned versions of patch level crate updates --- Cargo.lock | 367 +++++++++-------------------------------------------- 1 file changed, 59 insertions(+), 308 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 010c317c5..a3c36877f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,9 +92,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "block-buffer" @@ -159,9 +159,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.99" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" [[package]] name = "cfg-if" @@ -194,9 +194,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.5" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2020fa13af48afc65a9a87335bda648309ab3d154cd03c7ff95b378c7ed39c4" +checksum = "fbca90c87c2a04da41e95d1856e8bcd22f159bdbfa147314d2ce5218057b0e58" dependencies = [ "clap", ] @@ -210,7 +210,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.68", ] [[package]] @@ -340,17 +340,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "displaydoc" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "dunce" version = "1.0.4" @@ -359,9 +348,9 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "errno" @@ -578,7 +567,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbd06203b1a9b33a78c88252a625031b094d9e1b647260070c25b09910c0a804" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bstr", "gix-path", "libc", @@ -587,9 +576,9 @@ dependencies = [ [[package]] name = "gix-date" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "367ee9093b0c2b04fd04c5c7c8b6a1082713534eab537597ae343663a518fa99" +checksum = "9eed6931f21491ee0aeb922751bd7ec97b4b2fe8fbfedcb678e2a2dce5f3b8c0" dependencies = [ "bstr", "itoa", @@ -657,7 +646,7 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae6232f18b262770e343dcdd461c0011c9b9ae27f0c805e115012aa2b902c1b8" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bstr", "gix-features", "gix-path", @@ -690,7 +679,7 @@ version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e50e63df6c8d4137f7fb882f27643b3a9756c468a1a2cdbe1ce443010ca8778" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bstr", "btoi", "filetime", @@ -728,7 +717,7 @@ checksum = "999ce923619f88194171a67fb3e6d613653b8d4d6078b529b15a765da0edcc17" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.68", ] [[package]] @@ -791,9 +780,9 @@ dependencies = [ [[package]] name = "gix-path" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23623cf0f475691a6d943f898c4d0b89f5c1a2a64d0f92bce0e0322ee6528783" +checksum = "ca987128ffb056d732bd545db5db3d8b103d252fbf083c2567bb0796876619a4" dependencies = [ "bstr", "gix-trace", @@ -885,7 +874,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fddc27984a643b20dd03e97790555804f98cf07404e0e552c0ad8133266a79a1" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "gix-path", "libc", "windows-sys 0.52.0", @@ -1010,134 +999,14 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - -[[package]] -name = "icu_properties" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "idna" -version = "1.0.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ - "icu_normalizer", - "icu_properties", - "smallvec", - "utf8_iter", + "unicode-bidi", + "unicode-normalization", ] [[package]] @@ -1173,12 +1042,6 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" -[[package]] -name = "litemap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" - [[package]] name = "lock_api" version = "0.4.12" @@ -1191,9 +1054,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lua-src" @@ -1216,9 +1079,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -1231,18 +1094,18 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] [[package]] name = "mlua" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e340c022072f3208a4105458286f4985ba5355bfe243c3073afe45cbe9ecf491" +checksum = "d111deb18a9c9bd33e1541309f4742523bfab01d276bfa9a27519f6de9c11dc7" dependencies = [ "bstr", "mlua-sys", @@ -1254,9 +1117,9 @@ dependencies = [ [[package]] name = "mlua-sys" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5552e7e4e22ada0463dfdeee6caf6dc057a189fdc83136408a8f950a5e5c5540" +checksum = "a088ed0723df7567f569ba018c5d48c23c501f3878b190b04144dfa5ebfa8abc" dependencies = [ "cc", "cfg-if", @@ -1277,7 +1140,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.66", + "syn 2.0.68", ] [[package]] @@ -1328,7 +1191,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.1", + "redox_syscall 0.5.2", "smallvec", "windows-targets 0.52.5", ] @@ -1377,9 +1240,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -1410,11 +1273,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -1472,7 +1335,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.66", + "syn 2.0.68", "walkdir", ] @@ -1489,9 +1352,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustix" @@ -1499,7 +1362,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -1559,14 +1422,14 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.68", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "itoa", "ryu", @@ -1629,12 +1492,6 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "strsim" version = "0.11.1" @@ -1653,26 +1510,15 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "tempfile" version = "3.10.1" @@ -1712,7 +1558,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.68", ] [[package]] @@ -1748,21 +1594,11 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -1779,6 +1615,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-bom" version = "2.0.3" @@ -1802,27 +1644,15 @@ dependencies = [ [[package]] name = "url" -version = "2.5.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "utf8parse" version = "0.2.2" @@ -2034,82 +1864,3 @@ name = "winsafe" version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" - -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - -[[package]] -name = "yoke" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - -[[package]] -name = "zerofrom" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - -[[package]] -name = "zerovec" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] From d1244864af19417091f4be62a958e887abeca712 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 28 Jun 2024 11:22:42 +0300 Subject: [PATCH 31/32] chore(tooling): Update Git ignore list and sort for easier review --- .gitignore | 9 +++++---- .styluaignore | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index bc04e5075..fd5dbd464 100644 --- a/.gitignore +++ b/.gitignore @@ -88,17 +88,18 @@ sile-lua.1 .version .version-prev .tarball-version +.flake-version .built-subdirs *.so *.o -core/version.lua -core/features.lua -target/ completions/ +core/features.lua core/pathsetup.lua -src/embed.rs +core/version.lua src/embed-includes.rs +src/embed.rs src/sile-entry.sh +target/ *.asc # Nix symlink to builds diff --git a/.styluaignore b/.styluaignore index 3314e98a3..97633c2da 100644 --- a/.styluaignore +++ b/.styluaignore @@ -2,3 +2,4 @@ lua_modules lua-libraries core/features.lua core/pathsetup.lua +core/version.lua From b738c1c5cfa970c94188f7f7b7814aff28b33c39 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 28 Jun 2024 12:04:00 +0300 Subject: [PATCH 32/32] chore(release): 0.15.4 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ Cargo.lock | 6 +++--- Cargo.toml | 2 +- action.yml | 2 +- package.json | 2 +- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a63ca1e..64a8c8256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## [0.15.4](https://github.com/sile-typesetter/sile/compare/v0.15.3...v0.15.4) (2024-06-28) + + +### New Features + +* **build:** Add a developer target to test build dist without automake ([7bdaf16](https://github.com/sile-typesetter/sile/commit/7bdaf16564d96a0755ce112f448ee9eaeeaf7059)) +* **languages:** Add Brazilian Portuguese specific hyphenation points ([895c575](https://github.com/sile-typesetter/sile/commit/895c575e55974e4485701e04256516d130cc66f8)) +* **packages:** Biblatex data inheritance and field mapping ([646e3a4](https://github.com/sile-typesetter/sile/commit/646e3a4ef613d13db9491b87127f01d28f549b23)) +* **packages:** Support `[@string](https://github.com/string)` syntax in bibTeX bibliography ([63083ad](https://github.com/sile-typesetter/sile/commit/63083addace0925eb5d83803a1a86465d84cf797)), closes [#2051](https://github.com/sile-typesetter/sile/issues/2051) +* **packages:** Support `[@xdata](https://github.com/xdata)` entry type and xdata field in bibTeX bibliography ([ca906f0](https://github.com/sile-typesetter/sile/commit/ca906f0cfc882c5ad4ecb410f6aec001018b64a8)) +* **packages:** Support crossref field in bibTeX bibliography ([c4ae919](https://github.com/sile-typesetter/sile/commit/c4ae919ed72bce5740cb59809be8557dfe9b9321)), closes [#2021](https://github.com/sile-typesetter/sile/issues/2021) +* **tooling:** Add developer target to build Rust API docs ([384fa46](https://github.com/sile-typesetter/sile/commit/384fa463d71c6a7ecd24befd5670376874ce465f)) + + +### Bug Fixes + +* **build:** Avoid building sile binaries if not using them ([bbde8e6](https://github.com/sile-typesetter/sile/commit/bbde8e663dcd65262104c9bc32a10e28c3166e43)) +* **build:** Avoid the perceived need for an extra automake cycle in dist tarball ([f634a60](https://github.com/sile-typesetter/sile/commit/f634a606b8d6973e2815d7f769b0205d27ba0e3b)) +* **build:** Do not distribute main man page, requires Rust tooling ([26a6346](https://github.com/sile-typesetter/sile/commit/26a63463346a73eec38062984a8fec75761238b7)) +* **build:** Don't distribute distfiles list with configure option specific output ([b9d95fd](https://github.com/sile-typesetter/sile/commit/b9d95fdac11ed3e783520dd2b66690db52bad299)) +* **build:** Flag transitive dependency on Git if *not* using system LuaRocks ([ea2d501](https://github.com/sile-typesetter/sile/commit/ea2d501b65f1980c8b88b47dc9eedf6358f24453)) +* **languages:** Always set Fluent locale when setting `document.language` ([283fdc3](https://github.com/sile-typesetter/sile/commit/283fdc3062313394850cf5067c846db82f264c2e)) +* **packages:** Correct handling of ampersands and tildes in bibtex ([aaa613f](https://github.com/sile-typesetter/sile/commit/aaa613f636e6c892c70ea560a0815999ee617b2e)), closes [#2050](https://github.com/sile-typesetter/sile/issues/2050) [#1860](https://github.com/sile-typesetter/sile/issues/1860) +* **packages:** Ignore `[@preamble](https://github.com/preamble)` in bibTeX bibliography ([742a0c4](https://github.com/sile-typesetter/sile/commit/742a0c425787819c2cd14b950d5b7959669c86a6)), closes [#2051](https://github.com/sile-typesetter/sile/issues/2051) + ## [0.15.3](https://github.com/sile-typesetter/sile/compare/v0.15.2...v0.15.3) (2024-06-10) diff --git a/Cargo.lock b/Cargo.lock index a3c36877f..fcb495cc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1054,9 +1054,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lua-src" @@ -1474,7 +1474,7 @@ dependencies = [ [[package]] name = "sile" -version = "0.15.3" +version = "0.15.4" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 089786c2d..21e8bf2cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sile" -version = "0.15.3" +version = "0.15.4" edition = "2021" rust-version = "1.71.0" description = "Simon’s Improved Layout Engine" diff --git a/action.yml b/action.yml index 69d77a808..a001cb6ad 100644 --- a/action.yml +++ b/action.yml @@ -7,7 +7,7 @@ inputs: default: "" runs: using: docker - image: docker://ghcr.io/sile-typesetter/sile:v0.15.3 + image: docker://ghcr.io/sile-typesetter/sile:v0.15.4 entrypoint: sh args: - -c diff --git a/package.json b/package.json index 84d3d5d42..032996177 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sile", - "version": "0.15.3", + "version": "0.15.4", "description": "The SILE Typesetter", "main": "sile", "scripts": {