Preserve Unicode words during text wrapping

CSharpMath.Rendering.Text.Tests/TextLaTeXParserTests.cs

@@ -1,6 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using CSharpMath.Display.Displays;
+using CSharpMath.Rendering.BackEnd;
+using CSharpMath.Rendering.Text;
 using Xunit;
 
 namespace CSharpMath.Rendering.Text.Tests {
@@ -45,6 +48,82 @@ public void Text(params string[] text) {
         : new TextAtom.List(text.Select(t => new TextAtom.Text(t)).ToArray()), atom);
       Assert.Equal(input, TextLaTeXParser.TextAtomToLaTeX(atom).ToString());
     }
+
+    [Theory]
+    [InlineData("caf\u00E9")]
+    [InlineData("\u0421\u043B\u043E\u0432\u043E")]
+    [InlineData("e\u0301clair")]
+    [InlineData("e\u0301\u0327\u0308clair")]
+    public void UnicodeLettersAndCombiningMarksStayInOneWord(string word) {
+      Assert.Equal(new TextAtom.Text(word), Parse(word));
+    }
+
+    [Fact]
+    public void CjkCharactersRemainIndividuallyBreakable() {
+      Assert.Equal(
+        new TextAtom.List(new TextAtom[] {
+          new TextAtom.Text("\u4E2D"), new TextAtom.Text("\u6587")
+        }), Parse("\u4E2D\u6587"));
+    }
+
+    [Theory]
+    [InlineData("a\u200Bb")] // ZERO WIDTH SPACE: break opportunity
+    [InlineData("a\u00ADb")] // SOFT HYPHEN: conditional break opportunity
+    [InlineData("a\u2060b")] // WORD JOINER: no break opportunity
+    [InlineData("a\u200Db")] // ZERO WIDTH JOINER: format control
+    public void FormatControlsRoundTripWithoutBeingGlued(string input) {
+      Assert.Equal(input, TextLaTeXParser.TextAtomToLaTeX(Parse(input)).ToString());
+    }
+
+    [Fact]
+    public void SupplementaryLettersRemainWholeScalars() {
+      const string supplementaryLetter = "\U00010400\U00010401";
+      Assert.Equal(new TextAtom.Text(supplementaryLetter), Parse(supplementaryLetter));
+    }
+
+    [Theory]
+    [InlineData("\u0E01\u0E02")] // Thai
+    [InlineData("\u1780\u1781")] // Khmer
+    [InlineData("\u1000\u1001")] // Myanmar
+    public void ComplexScriptsRetainTypographyBreakOpportunities(string input) {
+      Assert.Equal(
+        new TextAtom.List(input.Select(c => (TextAtom)new TextAtom.Text(c.ToString())).ToArray()),
+        Parse(input));
+    }
+
+    [Theory]
+    [InlineData("e\u0301\u0E01")] // Latin + combining mark + Thai
+    [InlineData("e\u0301\u1780")] // Latin + combining mark + Khmer
+    [InlineData("e\u0301\u4E2D")] // Latin + combining mark + CJK
+    public void CombiningMarksDoNotGlueAcrossScriptBoundaries(string input) {
+      Assert.Equal(
+        new TextAtom.List(new TextAtom[] {
+          new TextAtom.Text(input.Substring(0, 2)),
+          new TextAtom.Text(input.Substring(2))
+        }), Parse(input));
+    }
+
+    [Fact]
+    public void PunctuationAndLongWordsKeepTheirExistingBoundaries() {
+      const string longWord = "supercalifragilisticexpialidocious";
+      Assert.Equal(
+        new TextAtom.List(new TextAtom[] {
+          new TextAtom.Text(longWord + ","), new TextAtom.Text("next")
+        }), Parse(longWord + ",next"));
+    }
+
+    [Theory]
+    [InlineData("caf\u00E9 next", "caf\u00E9")]
+    [InlineData("\u0421\u043B\u043E\u0432\u043E next", "\u0421\u043B\u043E\u0432\u043E")]
+    [InlineData("e\u0301clair next", "e\u0301clair")]
+    [InlineData("supercalifragilisticexpialidocious next", "supercalifragilisticexpialidocious")]
+    public void ConstrainedLayoutDoesNotSplitUnbreakableWords(string input, string word) {
+      var (relative, _) = TextTypesetter.Layout(
+        Parse(input), new Fonts(Array.Empty<Typography.OpenFont.Typeface>(), 12), 20);
+      Assert.Contains(
+        relative.Displays.OfType<TextRunDisplay<Fonts, Glyph>>(),
+        display => display.Run.Text.ToString() == word);
+    }
     [Theory]
     [InlineData(@"a\alpha a", @"a\alpha a", "a", "α", "a")]
     [InlineData(@"<\textless <", @"\textless \textless \textless ", "<", "<", "<")]
@@ -504,4 +583,4 @@ public void Error(string badInput, string expected) {
       Assert.Equal(expected.Replace("\r", null), actual);
     }
   }
-}
+}

CSharpMath.Rendering/Text/TextLaTeXParser.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Text;
 using Typography.TextBreak;
 
@@ -8,6 +9,107 @@ namespace CSharpMath.Rendering.Text {
   using Atom;
   using static CSharpMath.Atom.Result;
   public static class TextLaTeXParser {
+    // Typography.TextBreak historically emits a break for every letter outside
+    // its built-in Latin ranges.  Coalesce those breaks here using the Unicode
+    // categories supplied by .NET, while retaining the break opportunities for
+    // CJK ideographs (which are intentionally breakable between characters).
+    static void CoalesceUnicodeWordBreaks(string text, List<BreakAtInfo> breakList) {
+      for (int i = breakList.Count - 1; i >= 0; i--) {
+        int boundary = breakList[i].breakAt;
+        if (boundary <= 0 || boundary >= text.Length || !KeepUnicodeSequenceTogether(text, boundary))
+          continue;
+        breakList.RemoveAt(i);
+      }
+    }
+
+    static bool KeepUnicodeSequenceTogether(string text, int boundary) {
+      int beforeAt = boundary - 1;
+      if (char.IsLowSurrogate(text[beforeAt])) beforeAt--;
+      int afterAt = boundary;
+      // Typography never intentionally breaks a surrogate pair.  Do not
+      // manufacture a new decision at a half-surrogate if malformed input does
+      // reach this layer.
+      if (char.IsLowSurrogate(text[afterAt])) return false;
+      UnicodeCategory before = CharUnicodeInfo.GetUnicodeCategory(text, beforeAt);
+      UnicodeCategory after = CharUnicodeInfo.GetUnicodeCategory(text, afterAt);
+      bool beforeMark = before is UnicodeCategory.NonSpacingMark
+        or UnicodeCategory.SpacingCombiningMark or UnicodeCategory.EnclosingMark;
+      bool afterMark = after is UnicodeCategory.NonSpacingMark
+        or UnicodeCategory.SpacingCombiningMark or UnicodeCategory.EnclosingMark;
+      if (afterMark)
+        // UAX #14 LB9: combining marks inherit the line-break class of the
+        // preceding base. Walk over the complete mark run, not just one mark.
+        return IsLetterScalar(text, FindBaseScalarStart(text, beforeAt));
+      if (beforeMark && IsLetterScalar(text, afterAt))
+        return CanCoalesceLetter(text, FindBaseScalarStart(text, beforeAt))
+          && CanCoalesceLetter(text, afterAt);
+      return CanCoalesceLetter(text, beforeAt) && CanCoalesceLetter(text, afterAt);
+    }
+
+    static int FindPreviousScalarStart(string text, int scalarEndAt) =>
+      scalarEndAt > 0 && char.IsLowSurrogate(text[scalarEndAt])
+        ? scalarEndAt - 1 : scalarEndAt;
+
+    static int FindBaseScalarStart(string text, int scalarAt) {
+      int candidate = scalarAt;
+      while (candidate >= 0) {
+        UnicodeCategory category = CharUnicodeInfo.GetUnicodeCategory(text, candidate);
+        if (category is not (UnicodeCategory.NonSpacingMark
+          or UnicodeCategory.SpacingCombiningMark or UnicodeCategory.EnclosingMark))
+          return candidate;
+        candidate = FindPreviousScalarStart(text, candidate - 1);
+      }
+      return scalarAt;
+    }
+
+    static bool IsLetterScalar(string text, int scalarAt) {
+      UnicodeCategory category = CharUnicodeInfo.GetUnicodeCategory(text, scalarAt);
+      return category is UnicodeCategory.UppercaseLetter
+        or UnicodeCategory.LowercaseLetter or UnicodeCategory.TitlecaseLetter
+        or UnicodeCategory.ModifierLetter or UnicodeCategory.OtherLetter;
+    }
+
+    static bool CanCoalesceLetter(string text, int scalarAt) {
+      if (!IsLetterScalar(text, scalarAt)) return false;
+      // Typography's range metadata identifies scripts whose line breaking is
+      // dictionary- or character-based (Thai, CJK, kana, Hangul, etc.).  For
+      // ranges it does not know (notably Cyrillic), Unicode letters form words.
+      if (char.ConvertToUtf32(text, scalarAt) is int codepoint
+          && TryGetUnicodeRangeInfo(codepoint, out var range))
+        return !IsComplexBreakableRange(range.Name);
+      return true;
+    }
+
+    static bool TryGetUnicodeRangeInfo(int codepoint, out Typography.OpenFont.UnicodeRangeInfo range) {
+      if (UnicodeRangeFinder.GetUniCodeRangeFor(codepoint, out range, out _)) return true;
+      return Unicode13RangeInfoList.TryGetUnicodeRangeInfo(codepoint, out range);
+    }
+
+    static bool IsComplexBreakableRange(string name) =>
+      name.StartsWith("Thai", StringComparison.Ordinal)
+      || name.StartsWith("Lao", StringComparison.Ordinal)
+      || name.StartsWith("Khmer", StringComparison.Ordinal)
+      || name.StartsWith("Myanmar", StringComparison.Ordinal)
+      || name.StartsWith("CJK", StringComparison.Ordinal)
+      || name.StartsWith("Hiragana", StringComparison.Ordinal)
+      || name.StartsWith("Katakana", StringComparison.Ordinal)
+      || name.StartsWith("Hangul", StringComparison.Ordinal)
+      || name.StartsWith("Arabic", StringComparison.Ordinal)
+      || name.StartsWith("Hebrew", StringComparison.Ordinal)
+      || name.StartsWith("Devanagari", StringComparison.Ordinal)
+      || name.StartsWith("Bengali", StringComparison.Ordinal)
+      || name.StartsWith("Gurmukhi", StringComparison.Ordinal)
+      || name.StartsWith("Gujarati", StringComparison.Ordinal)
+      || name.StartsWith("Oriya", StringComparison.Ordinal)
+      || name.StartsWith("Tamil", StringComparison.Ordinal)
+      || name.StartsWith("Telugu", StringComparison.Ordinal)
+      || name.StartsWith("Kannada", StringComparison.Ordinal)
+      || name.StartsWith("Malayalam", StringComparison.Ordinal)
+      || name.StartsWith("Sinhala", StringComparison.Ordinal)
+      || name.StartsWith("Tibetan", StringComparison.Ordinal)
+      || name.StartsWith("Syriac", StringComparison.Ordinal)
+      || name.StartsWith("Ethiopic", StringComparison.Ordinal);
+
     /* //Paste this into the C# Interactive, fill <username> yourself
 #r "C:/Users/<username>/source/repos/CSharpMath/Typography/Build/NetStandard/Typography.TextBreak/bin/Debug/netstandard1.3/Typography.TextBreak.dll"
 using Typography.TextBreak;
@@ -59,6 +161,7 @@ public static Result<TextAtom> TextAtomFromLaTeX(string latexSource) {
       foreach (var engine in AdditionalBreakingEngines)
         breaker.AddBreakingEngine(engine);
       breaker.BreakWords(latexSource);
+      CoalesceUnicodeWordBreaks(latexSource, breakList);
 
       Result CheckDollarCount(int startAt, ref int endAt, TextAtomListBuilder atoms) {
         switch (dollarCount) {
@@ -504,4 +607,4 @@ public static StringBuilder TextAtomToLaTeX(TextAtom atom, StringBuilder? b = nu
       }
     }
   }
-}
+}