diff --git a/CSharpMath.Rendering.Tests/TextLeft/CapitalGreeks.png b/CSharpMath.Rendering.Tests/TextLeft/CapitalGreeks.png index 8affafc0..5833b41c 100644 Binary files a/CSharpMath.Rendering.Tests/TextLeft/CapitalGreeks.png and b/CSharpMath.Rendering.Tests/TextLeft/CapitalGreeks.png differ diff --git a/CSharpMath.Rendering.Tests/TextLeftInfiniteWidth/CapitalGreeks.png b/CSharpMath.Rendering.Tests/TextLeftInfiniteWidth/CapitalGreeks.png index 8affafc0..5833b41c 100644 Binary files a/CSharpMath.Rendering.Tests/TextLeftInfiniteWidth/CapitalGreeks.png and b/CSharpMath.Rendering.Tests/TextLeftInfiniteWidth/CapitalGreeks.png differ diff --git a/CSharpMath.Rendering.Tests/TextRight/CapitalGreeks.png b/CSharpMath.Rendering.Tests/TextRight/CapitalGreeks.png index 8affafc0..5833b41c 100644 Binary files a/CSharpMath.Rendering.Tests/TextRight/CapitalGreeks.png and b/CSharpMath.Rendering.Tests/TextRight/CapitalGreeks.png differ diff --git a/CSharpMath.Rendering.Tests/TextRightInfiniteWidth/CapitalGreeks.png b/CSharpMath.Rendering.Tests/TextRightInfiniteWidth/CapitalGreeks.png index 8affafc0..5833b41c 100644 Binary files a/CSharpMath.Rendering.Tests/TextRightInfiniteWidth/CapitalGreeks.png and b/CSharpMath.Rendering.Tests/TextRightInfiniteWidth/CapitalGreeks.png differ diff --git a/CSharpMath.Rendering.Text.Tests/TextLaTeXParserTests.cs b/CSharpMath.Rendering.Text.Tests/TextLaTeXParserTests.cs index 05b8459d..9031c8da 100644 --- a/CSharpMath.Rendering.Text.Tests/TextLaTeXParserTests.cs +++ b/CSharpMath.Rendering.Text.Tests/TextLaTeXParserTests.cs @@ -1,6 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; +using CSharpMath.Display.Displays; +using CSharpMath.Rendering.BackEnd; +using CSharpMath.Rendering.Text; using Xunit; namespace CSharpMath.Rendering.Text.Tests { @@ -45,6 +48,82 @@ public void Text(params string[] text) { : new TextAtom.List(text.Select(t => new TextAtom.Text(t)).ToArray()), atom); Assert.Equal(input, TextLaTeXParser.TextAtomToLaTeX(atom).ToString()); } + + [Theory] + [InlineData("caf\u00E9")] + [InlineData("\u0421\u043B\u043E\u0432\u043E")] + [InlineData("e\u0301clair")] + [InlineData("e\u0301\u0327\u0308clair")] + public void UnicodeLettersAndCombiningMarksStayInOneWord(string word) { + Assert.Equal(new TextAtom.Text(word), Parse(word)); + } + + [Fact] + public void CjkCharactersRemainIndividuallyBreakable() { + Assert.Equal( + new TextAtom.List(new TextAtom[] { + new TextAtom.Text("\u4E2D"), new TextAtom.Text("\u6587") + }), Parse("\u4E2D\u6587")); + } + + [Theory] + [InlineData("a\u200Bb")] // ZERO WIDTH SPACE: break opportunity + [InlineData("a\u00ADb")] // SOFT HYPHEN: conditional break opportunity + [InlineData("a\u2060b")] // WORD JOINER: no break opportunity + [InlineData("a\u200Db")] // ZERO WIDTH JOINER: format control + public void FormatControlsRoundTripWithoutBeingGlued(string input) { + Assert.Equal(input, TextLaTeXParser.TextAtomToLaTeX(Parse(input)).ToString()); + } + + [Fact] + public void SupplementaryLettersRemainWholeScalars() { + const string supplementaryLetter = "\U00010400\U00010401"; + Assert.Equal(new TextAtom.Text(supplementaryLetter), Parse(supplementaryLetter)); + } + + [Theory] + [InlineData("\u0E01\u0E02")] // Thai + [InlineData("\u1780\u1781")] // Khmer + [InlineData("\u1000\u1001")] // Myanmar + public void ComplexScriptsRetainTypographyBreakOpportunities(string input) { + Assert.Equal( + new TextAtom.List(input.Select(c => (TextAtom)new TextAtom.Text(c.ToString())).ToArray()), + Parse(input)); + } + + [Theory] + [InlineData("e\u0301\u0E01")] // Latin + combining mark + Thai + [InlineData("e\u0301\u1780")] // Latin + combining mark + Khmer + [InlineData("e\u0301\u4E2D")] // Latin + combining mark + CJK + public void CombiningMarksDoNotGlueAcrossScriptBoundaries(string input) { + Assert.Equal( + new TextAtom.List(new TextAtom[] { + new TextAtom.Text(input.Substring(0, 2)), + new TextAtom.Text(input.Substring(2)) + }), Parse(input)); + } + + [Fact] + public void PunctuationAndLongWordsKeepTheirExistingBoundaries() { + const string longWord = "supercalifragilisticexpialidocious"; + Assert.Equal( + new TextAtom.List(new TextAtom[] { + new TextAtom.Text(longWord + ","), new TextAtom.Text("next") + }), Parse(longWord + ",next")); + } + + [Theory] + [InlineData("caf\u00E9 next", "caf\u00E9")] + [InlineData("\u0421\u043B\u043E\u0432\u043E next", "\u0421\u043B\u043E\u0432\u043E")] + [InlineData("e\u0301clair next", "e\u0301clair")] + [InlineData("supercalifragilisticexpialidocious next", "supercalifragilisticexpialidocious")] + public void ConstrainedLayoutDoesNotSplitUnbreakableWords(string input, string word) { + var (relative, _) = TextTypesetter.Layout( + Parse(input), new Fonts(Array.Empty(), 12), 20); + Assert.Contains( + relative.Displays.OfType>(), + display => display.Run.Text.ToString() == word); + } [Theory] [InlineData(@"a\alpha a", @"a\alpha a", "a", "α", "a")] [InlineData(@"<\textless <", @"\textless \textless \textless ", "<", "<", "<")] @@ -504,4 +583,4 @@ public void Error(string badInput, string expected) { Assert.Equal(expected.Replace("\r", null), actual); } } -} \ No newline at end of file +} diff --git a/CSharpMath.Rendering/Text/TextLaTeXParser.cs b/CSharpMath.Rendering/Text/TextLaTeXParser.cs index 569f92ed..9d92e9f5 100644 --- a/CSharpMath.Rendering/Text/TextLaTeXParser.cs +++ b/CSharpMath.Rendering/Text/TextLaTeXParser.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; using Typography.TextBreak; @@ -8,6 +9,107 @@ namespace CSharpMath.Rendering.Text { using Atom; using static CSharpMath.Atom.Result; public static class TextLaTeXParser { + // Typography.TextBreak historically emits a break for every letter outside + // its built-in Latin ranges. Coalesce those breaks here using the Unicode + // categories supplied by .NET, while retaining the break opportunities for + // CJK ideographs (which are intentionally breakable between characters). + static void CoalesceUnicodeWordBreaks(string text, List breakList) { + for (int i = breakList.Count - 1; i >= 0; i--) { + int boundary = breakList[i].breakAt; + if (boundary <= 0 || boundary >= text.Length || !KeepUnicodeSequenceTogether(text, boundary)) + continue; + breakList.RemoveAt(i); + } + } + + static bool KeepUnicodeSequenceTogether(string text, int boundary) { + int beforeAt = boundary - 1; + if (char.IsLowSurrogate(text[beforeAt])) beforeAt--; + int afterAt = boundary; + // Typography never intentionally breaks a surrogate pair. Do not + // manufacture a new decision at a half-surrogate if malformed input does + // reach this layer. + if (char.IsLowSurrogate(text[afterAt])) return false; + UnicodeCategory before = CharUnicodeInfo.GetUnicodeCategory(text, beforeAt); + UnicodeCategory after = CharUnicodeInfo.GetUnicodeCategory(text, afterAt); + bool beforeMark = before is UnicodeCategory.NonSpacingMark + or UnicodeCategory.SpacingCombiningMark or UnicodeCategory.EnclosingMark; + bool afterMark = after is UnicodeCategory.NonSpacingMark + or UnicodeCategory.SpacingCombiningMark or UnicodeCategory.EnclosingMark; + if (afterMark) + // UAX #14 LB9: combining marks inherit the line-break class of the + // preceding base. Walk over the complete mark run, not just one mark. + return IsLetterScalar(text, FindBaseScalarStart(text, beforeAt)); + if (beforeMark && IsLetterScalar(text, afterAt)) + return CanCoalesceLetter(text, FindBaseScalarStart(text, beforeAt)) + && CanCoalesceLetter(text, afterAt); + return CanCoalesceLetter(text, beforeAt) && CanCoalesceLetter(text, afterAt); + } + + static int FindPreviousScalarStart(string text, int scalarEndAt) => + scalarEndAt > 0 && char.IsLowSurrogate(text[scalarEndAt]) + ? scalarEndAt - 1 : scalarEndAt; + + static int FindBaseScalarStart(string text, int scalarAt) { + int candidate = scalarAt; + while (candidate >= 0) { + UnicodeCategory category = CharUnicodeInfo.GetUnicodeCategory(text, candidate); + if (category is not (UnicodeCategory.NonSpacingMark + or UnicodeCategory.SpacingCombiningMark or UnicodeCategory.EnclosingMark)) + return candidate; + candidate = FindPreviousScalarStart(text, candidate - 1); + } + return scalarAt; + } + + static bool IsLetterScalar(string text, int scalarAt) { + UnicodeCategory category = CharUnicodeInfo.GetUnicodeCategory(text, scalarAt); + return category is UnicodeCategory.UppercaseLetter + or UnicodeCategory.LowercaseLetter or UnicodeCategory.TitlecaseLetter + or UnicodeCategory.ModifierLetter or UnicodeCategory.OtherLetter; + } + + static bool CanCoalesceLetter(string text, int scalarAt) { + if (!IsLetterScalar(text, scalarAt)) return false; + // Typography's range metadata identifies scripts whose line breaking is + // dictionary- or character-based (Thai, CJK, kana, Hangul, etc.). For + // ranges it does not know (notably Cyrillic), Unicode letters form words. + if (char.ConvertToUtf32(text, scalarAt) is int codepoint + && TryGetUnicodeRangeInfo(codepoint, out var range)) + return !IsComplexBreakableRange(range.Name); + return true; + } + + static bool TryGetUnicodeRangeInfo(int codepoint, out Typography.OpenFont.UnicodeRangeInfo range) { + if (UnicodeRangeFinder.GetUniCodeRangeFor(codepoint, out range, out _)) return true; + return Unicode13RangeInfoList.TryGetUnicodeRangeInfo(codepoint, out range); + } + + static bool IsComplexBreakableRange(string name) => + name.StartsWith("Thai", StringComparison.Ordinal) + || name.StartsWith("Lao", StringComparison.Ordinal) + || name.StartsWith("Khmer", StringComparison.Ordinal) + || name.StartsWith("Myanmar", StringComparison.Ordinal) + || name.StartsWith("CJK", StringComparison.Ordinal) + || name.StartsWith("Hiragana", StringComparison.Ordinal) + || name.StartsWith("Katakana", StringComparison.Ordinal) + || name.StartsWith("Hangul", StringComparison.Ordinal) + || name.StartsWith("Arabic", StringComparison.Ordinal) + || name.StartsWith("Hebrew", StringComparison.Ordinal) + || name.StartsWith("Devanagari", StringComparison.Ordinal) + || name.StartsWith("Bengali", StringComparison.Ordinal) + || name.StartsWith("Gurmukhi", StringComparison.Ordinal) + || name.StartsWith("Gujarati", StringComparison.Ordinal) + || name.StartsWith("Oriya", StringComparison.Ordinal) + || name.StartsWith("Tamil", StringComparison.Ordinal) + || name.StartsWith("Telugu", StringComparison.Ordinal) + || name.StartsWith("Kannada", StringComparison.Ordinal) + || name.StartsWith("Malayalam", StringComparison.Ordinal) + || name.StartsWith("Sinhala", StringComparison.Ordinal) + || name.StartsWith("Tibetan", StringComparison.Ordinal) + || name.StartsWith("Syriac", StringComparison.Ordinal) + || name.StartsWith("Ethiopic", StringComparison.Ordinal); + /* //Paste this into the C# Interactive, fill yourself #r "C:/Users//source/repos/CSharpMath/Typography/Build/NetStandard/Typography.TextBreak/bin/Debug/netstandard1.3/Typography.TextBreak.dll" using Typography.TextBreak; @@ -59,6 +161,7 @@ public static Result TextAtomFromLaTeX(string latexSource) { foreach (var engine in AdditionalBreakingEngines) breaker.AddBreakingEngine(engine); breaker.BreakWords(latexSource); + CoalesceUnicodeWordBreaks(latexSource, breakList); Result CheckDollarCount(int startAt, ref int endAt, TextAtomListBuilder atoms) { switch (dollarCount) { @@ -504,4 +607,4 @@ public static StringBuilder TextAtomToLaTeX(TextAtom atom, StringBuilder? b = nu } } } -} \ No newline at end of file +}