From 4e661de0f433c98227216d14b5876a5432ae6c4b Mon Sep 17 00:00:00 2001 From: Angel Peralta Date: Mon, 16 Nov 2026 20:30:13 -0500 Subject: [PATCH] core: classify one-cell characters outside the known ranges as :other char-type returned :cjk for every character outside its known ranges, so Greek, Cyrillic, arrows, math symbols and box drawing were all "CJK". The SDL2 and SDL3 frontends draw :cjk with the CJK font, which draws many of these full-width in a one-cell slot, so both frontends had to re-check the width themselves (#2395, #2396). char-type now returns :cjk only for wide characters (wide-char-p) and the new type :other for the rest. Frontends that pick fonts by type can draw :other with their monospace font; ones that ignore the type, or send unknown types to a default, behave as before. - char-type is exported from lem-core/display, with docstrings for the type and the function, so frontends and tests no longer need lem-core::char-type. - SDL3 draws :other with the latin fonts (which fall back to the CJK font and the others) and drops narrow-non-cjk-char-p. - SDL2 draws :other with the latin font when it has the glyph, else with the CJK font, as before. Tests: new tests/char-type.lisp covers latin, wide (:cjk), one-cell (:other) and the unchanged types, and that every result is a char-type. The SDL font tests now take the type from char-type. lem-tests, lem-sdl2/tests and lem-sdl3/tests pass, and an SDL3 frame of Greek, Cyrillic, math, CJK and box drawing is pixel-identical to before. --- frontends/sdl2/display.lisp | 18 ++++++++---------- frontends/sdl2/tests/font.lisp | 14 +++++++------- frontends/sdl3/display.lisp | 25 ++++++------------------- frontends/sdl3/tests/font.lisp | 14 +++++++------- lem-tests.asd | 1 + src/display/char-type.lisp | 13 +++++++++++-- src/internal-packages.lisp | 1 + tests/char-type.lisp | 31 +++++++++++++++++++++++++++++++ 8 files changed, 72 insertions(+), 45 deletions(-) create mode 100644 tests/char-type.lisp diff --git a/frontends/sdl2/display.lisp b/frontends/sdl2/display.lisp index 280fa71a16..a8994d47b7 100644 --- a/frontends/sdl2/display.lisp +++ b/frontends/sdl2/display.lisp @@ -205,14 +205,12 @@ e.g. -3 for NotoSansMono at 12pt). |descent| pixels sit below the baseline." (defun narrow-char-font (display type bold character) "Return the monospace font to draw CHARACTER, of char-type TYPE, with, or -NIL if CHARACTER is not a one-cell :cjk character or that font lacks the -glyph. Lem calls every character it does not otherwise know :cjk (Greek, -Cyrillic, arrows, math, box drawing), but the CJK font draws many of these -full-width while Lem gives them one cell, and on its own baseline. -Monospace fonts draw them to fit the cell." - (when (and (eq type :cjk) - character - (not (lem:wide-char-p character))) +NIL if TYPE is not :other or that font lacks the glyph. :other characters +(Greek, Cyrillic, arrows, math, box drawing) take one cell; the monospace +font draws them to fit it, while the CJK font draws many of them full-width. +SDL2_ttf has no fallback fonts, so a glyph the monospace font lacks is drawn +with the CJK font instead." + (when (and (eq type :other) character) (let ((font (if bold (display-latin-bold-font display) (display-latin-font display)))) @@ -224,7 +222,7 @@ Monospace fonts draw them to fit the cell." get-display-font), return that font's (TOP . BOTTOM) box-drawing band, to be stretched to the cell height. Otherwise return NIL." (and (box-drawing-char-p character) - (narrow-char-font display :cjk bold character) + (narrow-char-font display :other bold character) (font:font-box-drawing-band (display-font display) bold))) (defun font-ascent-of (ttf-font) @@ -233,7 +231,7 @@ its baseline this many pixels below the top of the surface." (sdl2-ffi.functions:ttf-font-ascent ttf-font)) (defmethod get-display-font ((display display) &key type bold character) - (check-type type lem-core::char-type) + (check-type type lem-core/display:char-type) (cond ((eq type :control) (display-latin-font display)) ((narrow-char-font display type bold character)) diff --git a/frontends/sdl2/tests/font.lisp b/frontends/sdl2/tests/font.lisp index 5a4ee11be3..3a2e48a317 100644 --- a/frontends/sdl2/tests/font.lisp +++ b/frontends/sdl2/tests/font.lisp @@ -84,15 +84,15 @@ (font:close-font font)))) (deftest box-drawing-characters-use-the-monospace-font - ;; Lem classifies U+2500..U+259F as :cjk; the CJK font draws them - ;; full-width in a one-cell slot, so lines and corners did not meet. + ;; U+2500..U+259F are :other; the CJK font drew them full-width in a + ;; one-cell slot, so lines and corners did not meet. (with-ttf () (call-with-display-for-font 14 (lambda (display) (flet ((font-for (character &key bold) (lem-sdl2/display:get-display-font display - :type :cjk + :type (lem-core/display:char-type character) :bold bold :character character))) (testing "box drawing and block elements use the latin fonts" @@ -130,9 +130,9 @@ (ok (null (lem-sdl2/display:display-box-drawing-band display #\日 nil))))))))) (deftest narrow-characters-use-the-monospace-font - ;; Lem classifies every character it does not otherwise know (Greek, - ;; Cyrillic, arrows, math) as :cjk, which drew them with the CJK font in - ;; its own style and, being taller, below the row's baseline. + ;; One-cell characters outside the latin ranges (Greek, Cyrillic, arrows, + ;; math) are :other; drawn with the CJK font they took its style and, + ;; being taller, sat below the row's baseline. (with-ttf () (call-with-display-for-font 14 @@ -140,7 +140,7 @@ (let ((font (lem-sdl2/display::display-font display))) (flet ((font-for (character &key bold) (lem-sdl2/display:get-display-font display - :type :cjk + :type (lem-core/display:char-type character) :bold bold :character character))) (testing "one-cell characters use the latin fonts" diff --git a/frontends/sdl3/display.lisp b/frontends/sdl3/display.lisp index 786575f64c..92be8ceeda 100644 --- a/frontends/sdl3/display.lisp +++ b/frontends/sdl3/display.lisp @@ -194,16 +194,6 @@ e.g. -3 for NotoSansMono at 12pt). |descent| pixels sit below the baseline." (and character (<= #x2500 (char-code character) #x259f))) -(defun narrow-non-cjk-char-p (type character) - "Return true if CHARACTER, of char-type TYPE, is one Lem classifies as -:cjk but gives a single cell: Greek, Cyrillic, arrows, math and other -symbols, box drawing. Lem calls every character it does not otherwise know -:cjk, but the CJK font draws these in its own (often full-width) style; the -monospace font draws them to fit the cell." - (and (eq type :cjk) - character - (not (lem:wide-char-p character)))) - (defun display-glyph-ascent (display ttf-font character) "Return the ascent of the font that draws CHARACTER when it is rendered with TTF-FONT: TTF-FONT's own, or that of the fallback font providing the @@ -224,15 +214,9 @@ box-drawing band, to be stretched to the cell height. Otherwise NIL." (font:font-box-drawing-band (display-font display) bold))) (defmethod get-display-font ((display display) &key type bold character) - (check-type type lem-core::char-type) + (check-type type lem-core/display:char-type) (cond ((eq type :control) (display-latin-font display)) - ((narrow-non-cjk-char-p type character) - ;; The latin fonts fall back to the CJK font and then the others - ;; for glyphs they lack. - (if bold - (display-latin-bold-font display) - (display-latin-font display))) ((eq type :icon) (or (and character (icon-font:icon-font character @@ -242,13 +226,16 @@ box-drawing band, to be stretched to the cell height. Otherwise NIL." (display-emoji-font display)) ((eq type :braille) (display-braille-font display)) + ;; :other characters (Greek, Cyrillic, symbols, box drawing) take one + ;; cell, so they use the monospace fonts, which fall back to the CJK + ;; font and then the others for glyphs they lack. (bold (case type - ((:latin :zero-width) (display-latin-bold-font display)) + ((:latin :other :zero-width) (display-latin-bold-font display)) (otherwise (display-cjk-bold-font display)))) (t (case type - ((:latin :zero-width) (display-latin-font display)) + ((:latin :other :zero-width) (display-latin-font display)) (otherwise (display-cjk-normal-font display)))))) (defmethod scaled-char-width ((display display) x) diff --git a/frontends/sdl3/tests/font.lisp b/frontends/sdl3/tests/font.lisp index df59d9d74d..43ba2fca2d 100644 --- a/frontends/sdl3/tests/font.lisp +++ b/frontends/sdl3/tests/font.lisp @@ -84,15 +84,15 @@ (font:close-font font)))) (deftest box-drawing-characters-use-the-monospace-font - ;; Lem classifies U+2500..U+259F as :cjk; the CJK font draws them - ;; full-width in a one-cell slot, so lines and corners did not meet. + ;; U+2500..U+259F are :other; the CJK font drew them full-width in a + ;; one-cell slot, so lines and corners did not meet. (with-ttf () (call-with-display-for-font 14 (lambda (display) (flet ((font-for (character &key bold) (lem-sdl3/display:get-display-font display - :type :cjk + :type (lem-core/display:char-type character) :bold bold :character character))) (testing "box drawing and block elements use the latin fonts" @@ -206,9 +206,9 @@ (font:font-box-drawing-band b nil)))))))))) (deftest narrow-characters-use-the-monospace-font - ;; Lem classifies every character it does not otherwise know (Greek, - ;; Cyrillic, arrows, math) as :cjk, which drew them with the CJK font in - ;; its own style and, being taller, below the row's baseline. + ;; One-cell characters outside the latin ranges (Greek, Cyrillic, arrows, + ;; math) are :other; drawn with the CJK font they took its style and, + ;; being taller, sat below the row's baseline. (with-ttf () (call-with-display-for-font 14 @@ -216,7 +216,7 @@ (let ((font (lem-sdl3/display::display-font display))) (flet ((font-for (character &key bold) (lem-sdl3/display:get-display-font display - :type :cjk + :type (lem-core/display:char-type character) :bold bold :character character))) (testing "one-cell characters use the latin fonts" diff --git a/lem-tests.asd b/lem-tests.asd index 1c1fc73fca..2f7c90d00a 100644 --- a/lem-tests.asd +++ b/lem-tests.asd @@ -58,6 +58,7 @@ (:file "language-mode") (:file "killring") (:file "string-width-utils") + (:file "char-type") (:file "syntax-test") (:file "syntax-scanner") (:file "buffer-list-test") diff --git a/src/display/char-type.lisp b/src/display/char-type.lisp index 23eadb68e7..1385c81740 100644 --- a/src/display/char-type.lisp +++ b/src/display/char-type.lisp @@ -1,7 +1,11 @@ (in-package :lem-core) (deftype char-type () - '(member :latin :cjk :braille :emoji :icon :control :zero-width)) + "The kind of a character, as returned by `char-type'. Frontends use it to +choose a font: :latin (ASCII and Latin-1 to Latin Extended-B), :cjk (wide +characters), :other (any other one-cell character), :braille, :emoji, :icon, +:control and :zero-width." + '(member :latin :cjk :other :braille :emoji :icon :control :zero-width)) (defun cjk-char-code-p (code) (or (<= #x4E00 code #x9FFF) @@ -33,6 +37,9 @@ (= code #xFEFF))) (defun char-type (char) + "Return CHAR's `char-type'. A character in none of the known ranges is +:cjk if it is wide (two cells, see `wide-char-p') and :other otherwise, +e.g. Greek, Cyrillic, arrows, math symbols and box drawing." (let ((code (char-code char))) (cond ((control-char char) :control) @@ -54,5 +61,7 @@ :latin) ((zero-width-char-code-p code) :zero-width) + ((wide-char-p char) + :cjk) (t - :cjk)))) + :other)))) diff --git a/src/internal-packages.lisp b/src/internal-packages.lisp index 6f417ab7c5..74c5b7ec94 100644 --- a/src/internal-packages.lisp +++ b/src/internal-packages.lisp @@ -1,6 +1,7 @@ (defpackage :lem-core/display (:use) (:export + :char-type :wrap-line-character :wrap-line-attribute :control-character-object diff --git a/tests/char-type.lisp b/tests/char-type.lisp new file mode 100644 index 0000000000..0a1e22a7d1 --- /dev/null +++ b/tests/char-type.lisp @@ -0,0 +1,31 @@ +(defpackage :lem-tests/char-type + (:use :cl :rove) + (:import-from :lem-core/display + :char-type)) +(in-package :lem-tests/char-type) + +(deftest latin-characters + (dolist (character '(#\a #\Z #\0 #\~ #\é #\ß #\ő #\ƀ)) + (ok (eq :latin (char-type character)) (format nil "~S" character)))) + +(deftest wide-characters-are-cjk + (dolist (character '(#\日 #\あ #\ア #\한 #\ㄱ #\。 #\A)) + (ok (eq :cjk (char-type character)) (format nil "~S" character)))) + +(deftest one-cell-characters-are-other + ;; These used to be :cjk, so frontends drew them with the CJK font, which + ;; draws many of them full-width in a one-cell slot. + (dolist (character '(#\α #\Ω #\Ж #\я #\→ #\∀ #\≤ #\─ #\│ #\█ #\★)) + (ok (eq :other (char-type character)) (format nil "~S" character)) + (ok (not (lem:wide-char-p character)) (format nil "~S is one cell" character)))) + +(deftest other-types-are-unchanged + (ok (eq :braille (char-type #\⠿))) + (ok (eq :zero-width (char-type (code-char #x200b)))) + (ok (eq :control (char-type (code-char 1)))) + (ok (eq :emoji (char-type (code-char #x1f600))))) + +(deftest char-type-results-are-of-type-char-type + (dolist (character (list #\a #\日 #\α #\⠿ (code-char #x200b) (code-char 1) + (code-char #x1f600))) + (ok (typep (char-type character) 'char-type) (format nil "~S" character))))