ack
e4f1307aa9
A character written in the ^...^ mnemonic form -- ^oe^, ^zz^, ^hello^ --
used to kill ktext outright: no message, no output, and an exit status
that looked like a crash because it was one. Only the forms that happen
to spell hexadecimal digits escaped, and they were right by luck rather
than by reading. Any caret form that names no character is now a located
error that quotes what was written.
A klammer may now sit inside square brackets and braces as it already
could inside parentheses, and may be followed by the punctuation that
ends a sentence: [@i word@], {@b word@}, @i word@'s. The rule is a fixed
list of characters, because none of them can occur inside a klammer name
and so none of them is ambiguous next to a delimiter. A letter still
cannot: "x@" is the named closing delimiter, so a letter before "@" has
two readings and no rule can separate them.
The abbreviations an author writes -- the hyphen form @i-word, a klammer
glued to its neighbouring punctuation -- are now recognised as author
conveniences only. They are not applied to @@ definitions or @@@ system
commands, which are written by someone who needs no such shorthand and
whose forms are already whole.
Also fixed, each of them silent: an exported shell function in the
environment broke every @eval in the document; a line break written with
@nl was destroyed when it passed through a klammer that generates a
table, printing as \textbackslash{}newline in the finished PDF; and
"kdesc --coverage" shortened its file column by comparing the spelling of
a path rather than the path, so a klammer set reached by a symbolic link
showed a full path where its neighbours showed a short one.
New in the distribution: ref/, a small klammerset that describes the
Klammermachine's own tables in Klammertext. It currently provides the
character tables -- the diacritics and the extended Latin letters -- as
klammers that render to any target, so the documentation of what can be
written cannot drift from what the machine accepts. It is early, and
will grow.
The Klammertext taxonomy now has a section on targets, describes the
@@@klammerset system command alongside the other three, and states the
precedence of "^" -- what a caret means before a letter, a digit, a
punctuation character, and a quote, in one ordered list.
(from dev 6594c7ff8c1f)
441 lines
16 KiB
C++
441 lines
16 KiB
C++
#include <fstream>
|
|
|
|
#include "util.h"
|
|
#include "character.h"
|
|
#include "precomposed.h"
|
|
#include "error.h"
|
|
#include "target.h"
|
|
#include "log.h"
|
|
#include "show.h"
|
|
|
|
// A base letter composed with a combining mark. The PRECOMPOSED (NFC)
|
|
// code point is emitted where one exists -- that is the byte sequence
|
|
// editors and input methods produce for direct entry, so ^a" and a typed
|
|
// ä compare equal (search, copy-paste); the tables formerly emitted
|
|
// base + combining mark, which renders identically but is a different
|
|
// string. Pairs with no precomposed form keep the combining fallback.
|
|
// The table is GENERATED: doc/make_precomposed_table.py -> precomposed.h.
|
|
static std::string compose(const std::string& base, const std::string& mark_cp)
|
|
{
|
|
auto it = precomposed.find(base + mark_cp);
|
|
if (it != precomposed.end()) return it->second;
|
|
return base + utf8char(std::stoi(mark_cp, nullptr, 16));
|
|
}
|
|
|
|
/*
|
|
inline
|
|
std::string klammertext_special_characters { "@|*^#" };
|
|
|
|
inline
|
|
std::string encoding_marker { "UU" };
|
|
|
|
inline
|
|
std::string diacritic_symbols = "-'`h~\"cbrdwa";
|
|
|
|
inline
|
|
std::string diacritic_symbols_order = "'`h~\"c-brdwa";
|
|
*/
|
|
|
|
std::string utf8char(int cp)
|
|
{
|
|
char c[5]={ 0x00,0x00,0x00,0x00,0x00 };
|
|
if (cp<=0x7F) {
|
|
c[0] = cp;
|
|
} else if(cp<=0x7FF) {
|
|
c[0] = (cp>>6)+192;
|
|
c[1] = (cp&63)+128;
|
|
} else if(0xd800<=cp && cp<=0xdfff) {
|
|
return "Invalid Unicode: " + std::to_string(cp);
|
|
} else if(cp<=0xFFFF) {
|
|
c[0] = (cp>>12)+224;
|
|
c[1]= ((cp>>6)&63)+128;
|
|
c[2]=(cp&63)+128;
|
|
} else if (cp<=0x10FFFF) {
|
|
c[0] = (cp>>18)+240;
|
|
c[1] = ((cp>>12)&63)+128;
|
|
c[2] = ((cp>>6)&63)+128;
|
|
c[3]=(cp&63)+128;
|
|
} else {
|
|
// Reachable since ^UUUU^ accepts 6 hex digits: FFFFFF > 10FFFF.
|
|
return "Invalid Unicode: " + std::to_string(cp);
|
|
}
|
|
return std::string(c);
|
|
}
|
|
|
|
|
|
bool valid_codepoint(int cp)
|
|
{
|
|
if (cp < 0 || cp > 0x10FFFF) return false;
|
|
if (0xD800 <= cp && cp <= 0xDFFF) return false; // surrogates
|
|
if (0xFDD0 <= cp && cp <= 0xFDEF) return false; // noncharacters
|
|
if ((cp & 0xFFFE) == 0xFFFE) return false; // U+xFFFE / U+xFFFF
|
|
return true;
|
|
}
|
|
|
|
std::string invalid_codepoint_reason(int cp)
|
|
{
|
|
if (cp < 0 || cp > 0x10FFFF)
|
|
return "beyond U+10FFFF, the top of the Unicode range";
|
|
if (0xD800 <= cp && cp <= 0xDFFF)
|
|
return "a surrogate (U+D800-U+DFFF), not a character";
|
|
return "a noncharacter, reserved by Unicode for internal use";
|
|
}
|
|
|
|
std::string unicode_hex_to_char(std::string s, int width=4) //, std::string marker)
|
|
{
|
|
(void)K::log(4, s);
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
std::regex re;
|
|
switch (width) {
|
|
case 2: re = hex2_re; break;
|
|
case 4: re = hex4_re; break;
|
|
case 5: re = hex5_re; break;
|
|
}
|
|
for (std::sregex_iterator p { s.begin(), s.end(), re }; p!= end; ++p) {
|
|
int codepoint = stoi((*p)[1].str(), nullptr, 16);
|
|
auto c = utf8char(codepoint);
|
|
std::regex hit_re { regex_escape((*p)[0]) };
|
|
result = std::regex_replace(result, hit_re, c);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
std::string process_diacritics(std::string s)
|
|
{
|
|
(void)K::log(4);
|
|
// The base may not be whitespace, a digit, or ASCII punctuation (the
|
|
// four ranges !-/ :-@ [-` {-~): a diacritic sits on a letter. Without
|
|
// the exclusion, ^ before a quoted punctuation character followed by a
|
|
// mark character composed nonsense -- ^-- became a hyphen with a macron
|
|
// instead of a literal hyphen before a hyphen. A multi-byte (non-ASCII)
|
|
// base is unaffected: its bytes are outside every excluded range.
|
|
std::regex diacritic_re("\\^([^\\s0-9!-/:-@\\[-`{-~])([" + diacritic_symbols + "])");
|
|
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), diacritic_re }; p!= end; ++p) {
|
|
std::regex hit { regex_escape((*p)[0].str()) };
|
|
std::string ch = (*p)[1].str();
|
|
std::string d = (*p)[2].str();
|
|
result = std::regex_replace(result, hit, compose(ch, diacritics[d].first));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// A delimited caret form left unresolved by every pass in encode(). The
|
|
// character class matches the nonascii Ktype's second alternative exactly:
|
|
// what the lexer accepts as a shape, encode() must either resolve or
|
|
// reject, or the two disagree about what is writable.
|
|
const std::regex unresolved_nonascii_re(R"(\^[A-Za-z0-9]{1,6}\^)");
|
|
|
|
std::string extended_latin_symbol_pattern()
|
|
{
|
|
std::string result {};
|
|
std::string sep = "";
|
|
for (const auto& nr : extended_latin_symbols) {
|
|
result += sep;
|
|
result += nr;
|
|
sep = "|";
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string process_extended_latin(std::string s)
|
|
{
|
|
(void)K::log(4);
|
|
std::regex diacritic_re("\\^(" + extended_latin_symbol_pattern() + ")\\^");
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), diacritic_re }; p!= end; ++p) {
|
|
std::regex hit { regex_escape((*p)[0].str()) };
|
|
std::string ch = (*p)[1].str();
|
|
result = std::regex_replace(result, hit, unicode_hex_to_char(extended_latin[ch].first));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string process_pinyin(std::string s)
|
|
{
|
|
(void)K::log(4);
|
|
std::regex pinyin_re(R"(\^([aeiou])([1-4]))");
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), pinyin_re }; p!= end; ++p) {
|
|
std::regex hit { regex_escape((*p)[0].str()) };
|
|
std::string vowel = (*p)[1].str();
|
|
std::string tone = (*p)[2].str();
|
|
result = std::regex_replace(result, hit,
|
|
compose(vowel, pinyin_tones[tone].first));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string process_unicode_codepoint(std::string s, const Locator& loc = Locator::none())
|
|
{
|
|
(void)K::log(4);
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), unicode_re }; p!= end; ++p) {
|
|
std::regex hit_re { regex_escape((*p)[0].str()) };
|
|
// Convert the captured hex DIRECTLY: unicode_hex_to_char() re-scans
|
|
// its argument at a fixed width, and its 4-digit default truncated a
|
|
// 5-digit code point to its first four digits (^13000^ rendered as
|
|
// U+1300 followed by a literal "0").
|
|
int cp = std::stoi((*p)[1].str(), nullptr, 16);
|
|
// An invalid code point is a located ERROR, not text in the document
|
|
// (the no-warn-and-continue policy): a generator's typo must not
|
|
// produce a finished-looking document with an apology embedded
|
|
// where a character should be.
|
|
if (!valid_codepoint(cp)) {
|
|
throw Parsing_error("Invalid Unicode code point \"" + (*p)[0].str()
|
|
+ "\": " + invalid_codepoint_reason(cp), loc);
|
|
}
|
|
// A code point that lands on a Klammertext structural character
|
|
// (@ | # ^ : *) is hidden as its KTESC marker, exactly as the
|
|
// quoted form of that character would be -- a bare "@" from
|
|
// ^0040^ would otherwise be re-interpreted when the text is
|
|
// re-katomized (@document :text, the @eval read-back).
|
|
result = std::regex_replace(result, hit_re,
|
|
hide_structural_characters(utf8char(cp)));
|
|
}
|
|
return result;
|
|
|
|
}
|
|
|
|
// Old xhide/xrestore/hide/restore functions removed. The ^X mechanism
|
|
// is handled by the katomizer (katom_t::special) and the general KTESC
|
|
// escape mechanism in Target::escape_text/resolve_escapes.
|
|
|
|
|
|
|
|
std::string encode(const std::string& s, const Locator& loc)
|
|
{
|
|
if (s.find("^") == std::string::npos)
|
|
return s;
|
|
|
|
(void)(void)K::log(3);
|
|
std::string result = s;
|
|
bool dbg = verbose_level > 3;
|
|
|
|
std::string lit_start = "__LITSTART__";
|
|
std::string lit_end = "__LITEND__";
|
|
|
|
result = string_replace(result, "^'", lit_start);
|
|
result = string_replace(result, "'^", lit_end);
|
|
|
|
if (result.find("^") == std::string::npos)
|
|
return s;
|
|
|
|
if (dbg) std::cout << "start: " << result << "\n";
|
|
|
|
result = process_extended_latin(result);
|
|
if (dbg) std::cout << "extended_latin: " << result << "\n";
|
|
|
|
result = process_unicode_codepoint(result, loc);
|
|
if (dbg) std::cout << "unicode: " << result << "\n";
|
|
|
|
result = process_diacritics(result);
|
|
if (dbg) std::cout << "diacrit: " << result << "\n";
|
|
|
|
result = process_pinyin(result);
|
|
if (dbg) std::cout << "pinyin: " << result << "\n";
|
|
|
|
// A "^...^" form that survives every pass above resolves to no
|
|
// character: it is not in the extended-Latin table, not a code point,
|
|
// not a diacritic, not a pinyin tone. That is an ERROR here rather
|
|
// than text in the document -- the same rule as an invalid code point
|
|
// a few lines up, and for the same reason: a typo must not reach a
|
|
// finished-looking document as literal "^zz^". Only the DELIMITED
|
|
// form is checked. The two-character form (^a, ^-) is deliberately
|
|
// exempt: quoted punctuation resolves later, in hide_quoted_punctuation().
|
|
if (std::regex_search(result, unresolved_nonascii_re)) {
|
|
std::smatch hit {};
|
|
std::regex_search(result, hit, unresolved_nonascii_re);
|
|
throw Parsing_error("\"" + hit[0].str() + "\" is not a character code. "
|
|
"It names no extended-Latin mnemonic and is not a "
|
|
"Unicode code point. The forms are:\n\n"
|
|
" ^oe^ a mnemonic (kdesc -c lists them)\n"
|
|
" ^263A^ a code point, 4 to 6 hex digits\n"
|
|
" ^o\" a diacritic on a base letter\n",
|
|
loc);
|
|
}
|
|
|
|
result = string_replace(result, lit_start, "^'");
|
|
result = string_replace(result, lit_end, "'^");
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
|
|
// Decode
|
|
|
|
|
|
// Display
|
|
|
|
void write_kt_example_file(std::stringstream& kt, std::string kt_filename)
|
|
{
|
|
kt << "|| @line@\n";
|
|
kt << "@\n";
|
|
std::cout << "Writing " << kt_filename << "...";
|
|
std::ofstream out(kt_filename);
|
|
out << kt.str();
|
|
out.close();
|
|
std::cout << "done\n";
|
|
}
|
|
|
|
|
|
/*
|
|
void diacritics_examples(const std::string& kt_filename)
|
|
{
|
|
if (diacritics.size() != diacritic_symbols_order.size()) {
|
|
//throw Internal_error("Mismatch between diacritics order list and their definitions");
|
|
std::cout << "Mismatch error\n";
|
|
}
|
|
|
|
std::stringstream kt {};
|
|
bool write_kt_file = kt_filename.size() != 0;
|
|
if (write_kt_file)
|
|
kt << "@table :caption Diacritics (with typical base characters) |\n"
|
|
<< " @i-Displayed | @i-Written | @i-Name\n";
|
|
else
|
|
std::cout << boldblack
|
|
<< "\nDiacritics (with typical base characters)\n\n" << black;
|
|
|
|
std::string line_sep = "|| @line@ ";
|
|
for (char symbol : diacritic_symbols_order) {
|
|
std::string sym { symbol };
|
|
auto [code, name] = diacritics[sym];
|
|
std::string letter = diacritic_example_letter[sym];
|
|
std::string written = "^" + letter + sym;
|
|
if (sym == "~") {
|
|
written = "^" + letter + "=7e=";
|
|
}
|
|
if (write_kt_file) {
|
|
std::string display = "^" + letter + sym;
|
|
kt << line_sep << display << " | @t ^" << written << " @ | " << name << "\n";
|
|
line_sep = "|| ";
|
|
}
|
|
else {
|
|
std::cout << " " << letter << unicode_hex_to_char(code)
|
|
<< " " << "^" << letter << symbol << " " << name << "\n";
|
|
}
|
|
}
|
|
if (write_kt_file)
|
|
write_kt_example_file(kt, kt_filename);
|
|
}
|
|
*/
|
|
|
|
/*
|
|
void extended_latin_examples(const std::string& kt_filename)
|
|
{
|
|
std::stringstream kt {};
|
|
bool write_kt_file = kt_filename.size() != 0;
|
|
std::string title = "Extended Latin characters and ligatures";
|
|
if (write_kt_file)
|
|
kt << "@table :caption " << title << " |\n"
|
|
<< " @i-Displayed | @i-Written | @i-Name\n";
|
|
else
|
|
std::cout << "\n" << boldblack << title << black << "\n\n";
|
|
|
|
std::string line_sep = "|| @line@ ";
|
|
for (const std::string& symbol : extended_latin_symbols) {
|
|
auto [code, name] = extended_latin[symbol];
|
|
if (write_kt_file) {
|
|
std::string coded = "^" + symbol + "^";
|
|
std::string file_literal = "^^ #- " + symbol + " #- ^^";
|
|
kt << line_sep << coded << " | @t " << file_literal << " @ | " << name << "\n";
|
|
line_sep = "|| ";
|
|
}
|
|
else {
|
|
std::string screen_literal = "^" + symbol + "^";
|
|
std::cout << " " << unicode_hex_to_char(code)
|
|
<< " " << std::setw(4) << std::left << screen_literal << " " << name << "\n";
|
|
}
|
|
}
|
|
if (write_kt_file)
|
|
write_kt_example_file(kt, kt_filename);
|
|
}
|
|
*/
|
|
|
|
void pinyin_examples(const std::string& kt_filename)
|
|
{
|
|
std::stringstream kt {};
|
|
bool write_kt_file = kt_filename.size() != 0;
|
|
if (write_kt_file)
|
|
kt << "@table :caption Mandarin pinyin tones (using vowel ``a'') |\n"
|
|
<< " @i-Displayed | @i-Written | @i-Name\n";
|
|
else
|
|
std::cout << boldblack
|
|
<< "\nMandarin pinyin tones (using vowel \"a\")\n\n" << black;
|
|
|
|
std::string line_sep = "|| @line@ ";
|
|
for (auto [symbol, codename] : pinyin_tones) {
|
|
auto [code, name] = codename;
|
|
// std::string hat_code = "^a" + code;
|
|
std::string literal = "^a" + symbol;
|
|
if (write_kt_file) {
|
|
kt << line_sep << literal << " | @t ^" << literal << " @ | " << name << "\n";
|
|
line_sep = "|| ";
|
|
} else
|
|
std::cout << " a" << unicode_hex_to_char(code)
|
|
<< " "<< std::setw(4) << std::left << literal << " " << name << "\n";
|
|
}
|
|
if (write_kt_file)
|
|
write_kt_example_file(kt, kt_filename);
|
|
}
|
|
|
|
/*
|
|
Outside my window
|
|
crows build a nest in a tree...
|
|
slowly... carefully.
|
|
*/
|
|
|
|
void unicode_examples(const std::string& kt_filename)
|
|
{
|
|
std::stringstream kt {};
|
|
bool write_kt_file = kt_filename.size() != 0;
|
|
|
|
std::cout << boldblack << "\nIncluding Unicode in Klammertext files\n\n" << black
|
|
<< "To specify a wide variety of different letter forms, the Unicode Standard\n"
|
|
<< "defines a large set of \"code points\" organized by groups. To include a\n"
|
|
<< "Unicode character in Klammertext, you surround its hexadecimal number with\n"
|
|
<< "the \"^\" character. For example, the Cyrillic captial letter \"I\" is \"И\",\n"
|
|
<< "included in Klammertext as ^0418^.\n\n"
|
|
<< "To search for Unicode characters by their description, see:\n"
|
|
<< " https://www.fileformat.info/info/unicode/char/search.htm\n";
|
|
|
|
std::cout << "\n⟐ Examples of Chinese characters\n\n"
|
|
<< " Pinyin\n";
|
|
for (auto chr : dao) {
|
|
std::string code = "^" + chr[1] + "^";
|
|
std::cout << " " << process_unicode_codepoint(code) << " "
|
|
<< code << " "
|
|
<< chr[0] << "\n";
|
|
}
|
|
|
|
std::cout << "\n⟐ Examples of Egyptian hieroglyphs\n\n";
|
|
std::cout << std::setw(16) << " " << "Sound Gardiner\n";
|
|
for (auto glyph : ptolemy) {
|
|
std::cout << " " << std::setw(4) << process_unicode_codepoint("^" + glyph[2] + "^") << " "
|
|
<< "^" + glyph[2] + "^ "
|
|
<< std::setw(4)
|
|
<< glyph[1] << " "
|
|
<< std::setw(4)
|
|
<< glyph[0] << "\n";
|
|
}
|
|
std::cout << "\nSee: https://en.wikipedia.org/wiki/List_of_Egyptian_hieroglyphs\n"
|
|
<< "The Unicode codepoint in the second column of the \"List of hieroglyphs\"\n"
|
|
<< "table is the number in the ^...^ code used by Klammertext.\n\n";
|
|
}
|
|
|
|
void show_special_characters()
|
|
{
|
|
std::cout << std::setfill(' ');
|
|
// diacritics_examples();
|
|
// extended_latin_examples();
|
|
pinyin_examples();
|
|
}
|