ackandClaude Fable 5.1
93d649e23c
extended_latin_symbol_pattern() assigned a string literal to a std::string inside its loop, for which gcc 12 reports a bogus memcpy overlap at -O3. The separator is a const char* now. No behaviour change. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
273 lines
9.9 KiB
C++
273 lines
9.9 KiB
C++
#include <fstream>
|
|
|
|
#include "util.h"
|
|
#include "character.h"
|
|
#include "precomposed.h"
|
|
#include "error.h"
|
|
#include "target.h"
|
|
#include "log.h"
|
|
#include "show.h"
|
|
|
|
// A base letter composed with a combining mark. The PRECOMPOSED (NFC)
|
|
// code point is emitted where one exists -- that is the byte sequence
|
|
// editors and input methods produce for direct entry, so ^a" and a typed
|
|
// ä compare equal (search, copy-paste); the tables formerly emitted
|
|
// base + combining mark, which renders identically but is a different
|
|
// string. Pairs with no precomposed form keep the combining fallback.
|
|
// The table is GENERATED: doc/make_precomposed_table.py -> precomposed.h.
|
|
static std::string compose(const std::string& base, const std::string& mark_cp)
|
|
{
|
|
auto it = precomposed.find(base + mark_cp);
|
|
if (it != precomposed.end()) return it->second;
|
|
return base + utf8char(std::stoi(mark_cp, nullptr, 16));
|
|
}
|
|
|
|
/*
|
|
inline
|
|
std::string klammertext_special_characters { "@|*^#" };
|
|
|
|
inline
|
|
std::string encoding_marker { "UU" };
|
|
|
|
inline
|
|
std::string diacritic_symbols = "-'`h~\"cbrdwa";
|
|
|
|
inline
|
|
std::string diacritic_symbols_order = "'`h~\"c-brdwa";
|
|
*/
|
|
|
|
std::string utf8char(int cp)
|
|
{
|
|
char c[5]={ 0x00,0x00,0x00,0x00,0x00 };
|
|
if (cp<=0x7F) {
|
|
c[0] = cp;
|
|
} else if(cp<=0x7FF) {
|
|
c[0] = (cp>>6)+192;
|
|
c[1] = (cp&63)+128;
|
|
} else if(0xd800<=cp && cp<=0xdfff) {
|
|
return "Invalid Unicode: " + std::to_string(cp);
|
|
} else if(cp<=0xFFFF) {
|
|
c[0] = (cp>>12)+224;
|
|
c[1]= ((cp>>6)&63)+128;
|
|
c[2]=(cp&63)+128;
|
|
} else if (cp<=0x10FFFF) {
|
|
c[0] = (cp>>18)+240;
|
|
c[1] = ((cp>>12)&63)+128;
|
|
c[2] = ((cp>>6)&63)+128;
|
|
c[3]=(cp&63)+128;
|
|
} else {
|
|
// Reachable since ^UUUU^ accepts 6 hex digits: FFFFFF > 10FFFF.
|
|
return "Invalid Unicode: " + std::to_string(cp);
|
|
}
|
|
return std::string(c);
|
|
}
|
|
|
|
|
|
bool valid_codepoint(int cp)
|
|
{
|
|
if (cp < 0 || cp > 0x10FFFF) return false;
|
|
if (0xD800 <= cp && cp <= 0xDFFF) return false; // surrogates
|
|
if (0xFDD0 <= cp && cp <= 0xFDEF) return false; // noncharacters
|
|
if ((cp & 0xFFFE) == 0xFFFE) return false; // U+xFFFE / U+xFFFF
|
|
return true;
|
|
}
|
|
|
|
std::string invalid_codepoint_reason(int cp)
|
|
{
|
|
if (cp < 0 || cp > 0x10FFFF)
|
|
return "beyond U+10FFFF, the top of the Unicode range";
|
|
if (0xD800 <= cp && cp <= 0xDFFF)
|
|
return "a surrogate (U+D800-U+DFFF), not a character";
|
|
return "a noncharacter, reserved by Unicode for internal use";
|
|
}
|
|
|
|
std::string unicode_hex_to_char(std::string s, int width=4) //, std::string marker)
|
|
{
|
|
(void)K::log(4, s);
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
std::regex re;
|
|
switch (width) {
|
|
case 2: re = hex2_re; break;
|
|
case 4: re = hex4_re; break;
|
|
case 5: re = hex5_re; break;
|
|
}
|
|
for (std::sregex_iterator p { s.begin(), s.end(), re }; p!= end; ++p) {
|
|
int codepoint = stoi((*p)[1].str(), nullptr, 16);
|
|
auto c = utf8char(codepoint);
|
|
std::regex hit_re { regex_escape((*p)[0]) };
|
|
result = std::regex_replace(result, hit_re, c);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
std::string process_diacritics(std::string s)
|
|
{
|
|
(void)K::log(4);
|
|
// The base may not be whitespace, a digit, or ASCII punctuation (the
|
|
// four ranges !-/ :-@ [-` {-~): a diacritic sits on a letter. Without
|
|
// the exclusion, ^ before a quoted punctuation character followed by a
|
|
// mark character composed nonsense -- ^-- became a hyphen with a macron
|
|
// instead of a literal hyphen before a hyphen. A multi-byte (non-ASCII)
|
|
// base is unaffected: its bytes are outside every excluded range.
|
|
std::regex diacritic_re("\\^([^\\s0-9!-/:-@\\[-`{-~])([" + diacritic_symbols + "])");
|
|
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), diacritic_re }; p!= end; ++p) {
|
|
std::regex hit { regex_escape((*p)[0].str()) };
|
|
std::string ch = (*p)[1].str();
|
|
std::string d = (*p)[2].str();
|
|
result = std::regex_replace(result, hit, compose(ch, diacritics[d].first));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// A delimited caret form left unresolved by every pass in encode(). The
|
|
// character class matches the nonascii Ktype's second alternative exactly:
|
|
// what the lexer accepts as a shape, encode() must either resolve or
|
|
// reject, or the two disagree about what is writable.
|
|
const std::regex unresolved_nonascii_re(R"(\^[A-Za-z0-9]{1,6}\^)");
|
|
|
|
std::string extended_latin_symbol_pattern()
|
|
{
|
|
std::string result {};
|
|
// A pointer, not a std::string: gcc 12 (the container's compiler) warns
|
|
// -Wrestrict on assigning a string literal to a std::string here -- a
|
|
// false positive of that release -- and the separator needs no storage.
|
|
const char* sep = "";
|
|
for (const auto& nr : extended_latin_symbols) {
|
|
result += sep;
|
|
result += nr;
|
|
sep = "|";
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string process_extended_latin(std::string s)
|
|
{
|
|
(void)K::log(4);
|
|
std::regex diacritic_re("\\^(" + extended_latin_symbol_pattern() + ")\\^");
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), diacritic_re }; p!= end; ++p) {
|
|
std::regex hit { regex_escape((*p)[0].str()) };
|
|
std::string ch = (*p)[1].str();
|
|
result = std::regex_replace(result, hit, unicode_hex_to_char(extended_latin[ch].first));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string process_pinyin(std::string s)
|
|
{
|
|
(void)K::log(4);
|
|
std::regex pinyin_re(R"(\^([aeiou])([1-4]))");
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), pinyin_re }; p!= end; ++p) {
|
|
std::regex hit { regex_escape((*p)[0].str()) };
|
|
std::string vowel = (*p)[1].str();
|
|
std::string tone = (*p)[2].str();
|
|
result = std::regex_replace(result, hit,
|
|
compose(vowel, pinyin_tones[tone].first));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string process_unicode_codepoint(std::string s, const Locator& loc)
|
|
{
|
|
(void)K::log(4);
|
|
std::string result {s};
|
|
std::sregex_iterator end {};
|
|
for (std::sregex_iterator p { s.begin(), s.end(), unicode_re }; p!= end; ++p) {
|
|
std::regex hit_re { regex_escape((*p)[0].str()) };
|
|
// Convert the captured hex DIRECTLY: unicode_hex_to_char() re-scans
|
|
// its argument at a fixed width, and its 4-digit default truncated a
|
|
// 5-digit code point to its first four digits (^13000^ rendered as
|
|
// U+1300 followed by a literal "0").
|
|
int cp = std::stoi((*p)[1].str(), nullptr, 16);
|
|
// An invalid code point is a located ERROR, not text in the document
|
|
// (the no-warn-and-continue policy): a generator's typo must not
|
|
// produce a finished-looking document with an apology embedded
|
|
// where a character should be.
|
|
if (!valid_codepoint(cp)) {
|
|
throw Parsing_error("Invalid Unicode code point \"" + (*p)[0].str()
|
|
+ "\": " + invalid_codepoint_reason(cp), loc);
|
|
}
|
|
// A code point that lands on a Klammertext structural character
|
|
// (@ | # ^ : *) is hidden as its KTESC marker, exactly as the
|
|
// quoted form of that character would be -- a bare "@" from
|
|
// ^0040^ would otherwise be re-interpreted when the text is
|
|
// re-katomized (@document :text, the @eval read-back).
|
|
result = std::regex_replace(result, hit_re,
|
|
hide_structural_characters(utf8char(cp)));
|
|
}
|
|
return result;
|
|
|
|
}
|
|
|
|
// Old xhide/xrestore/hide/restore functions removed. The ^X mechanism
|
|
// is handled by the katomizer (katom_t::special) and the general KTESC
|
|
// escape mechanism in Target::escape_text/resolve_escapes.
|
|
|
|
|
|
|
|
std::string encode(const std::string& s, const Locator& loc)
|
|
{
|
|
if (s.find("^") == std::string::npos)
|
|
return s;
|
|
|
|
(void)(void)K::log(3);
|
|
std::string result = s;
|
|
bool dbg = verbose_level > 3;
|
|
|
|
std::string lit_start = "__LITSTART__";
|
|
std::string lit_end = "__LITEND__";
|
|
|
|
result = string_replace(result, "^'", lit_start);
|
|
result = string_replace(result, "'^", lit_end);
|
|
|
|
if (result.find("^") == std::string::npos)
|
|
return s;
|
|
|
|
if (dbg) std::cout << "start: " << result << "\n";
|
|
|
|
result = process_extended_latin(result);
|
|
if (dbg) std::cout << "extended_latin: " << result << "\n";
|
|
|
|
result = process_unicode_codepoint(result, loc);
|
|
if (dbg) std::cout << "unicode: " << result << "\n";
|
|
|
|
result = process_diacritics(result);
|
|
if (dbg) std::cout << "diacrit: " << result << "\n";
|
|
|
|
result = process_pinyin(result);
|
|
if (dbg) std::cout << "pinyin: " << result << "\n";
|
|
|
|
// A "^...^" form that survives every pass above resolves to no
|
|
// character: it is not in the extended-Latin table, not a code point,
|
|
// not a diacritic, not a pinyin tone. That is an ERROR here rather
|
|
// than text in the document -- the same rule as an invalid code point
|
|
// a few lines up, and for the same reason: a typo must not reach a
|
|
// finished-looking document as literal "^zz^". Only the DELIMITED
|
|
// form is checked. The two-character form (^a, ^-) is deliberately
|
|
// exempt: quoted punctuation resolves later, in hide_quoted_punctuation().
|
|
if (std::regex_search(result, unresolved_nonascii_re)) {
|
|
std::smatch hit {};
|
|
std::regex_search(result, hit, unresolved_nonascii_re);
|
|
throw Parsing_error("\"" + hit[0].str() + "\" is not a character code. "
|
|
"It names no extended-Latin mnemonic and is not a "
|
|
"Unicode code point. The forms are:\n\n"
|
|
" ^oe^ a mnemonic (kdesc -c lists them)\n"
|
|
" ^263A^ a code point, 4 to 6 hex digits\n"
|
|
" ^o\" a diacritic on a base letter\n",
|
|
loc);
|
|
}
|
|
|
|
result = string_replace(result, lit_start, "^'");
|
|
result = string_replace(result, lit_end, "'^");
|
|
|
|
return result;
|
|
}
|