Andy Kopra
37b6ba1c4f
Typographic transforms (---, quote pairs, ~) no longer touch verbatim text: @c/@code/@source_listing content and ^'...'^ spans show exactly the characters written. "^" before any punctuation character quotes it in every target (the apostrophe excepted: ^' opens a literal span), with the new :resolve option on @@@target declaring per-target renderings. The ^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range. The html output and transform spellings are polyglot (XML-valid), in preparation for an EPUB target. New suites: transform_test, character_test (engine), typography_test (SKS). (from dev 07ce5ea86a0a)
613 lines
18 KiB
C++
613 lines
18 KiB
C++
#include <stdlib.h>
|
|
#include <cstdio>
|
|
#include <algorithm>
|
|
#include <cctype>
|
|
#include <iterator>
|
|
#include <set>
|
|
#include <sstream>
|
|
#include <utility>
|
|
#include <regex>
|
|
|
|
#include "util.h"
|
|
#include "show.h"
|
|
|
|
std::string trim_left(const std::string& s)
|
|
{
|
|
std::string result = s;
|
|
result.erase(result.begin(), std::find_if(result.begin(), result.end(), [](unsigned char ch) {
|
|
return !std::isspace(ch);
|
|
}));
|
|
return result;
|
|
}
|
|
|
|
std::string trim_right(const std::string& s)
|
|
{
|
|
std::string result = s;
|
|
result.erase(std::find_if(result.rbegin(), result.rend(), [](unsigned char ch) {
|
|
return !std::isspace(ch);
|
|
}).base(), result.end());
|
|
return result;
|
|
}
|
|
|
|
std::string trim(std::string s)
|
|
{
|
|
return trim_left(trim_right(std::move(s)));
|
|
}
|
|
|
|
std::string trim_char_left(std::string s, char remove)
|
|
{
|
|
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [&](char c) { return c != remove; }));
|
|
return s;
|
|
}
|
|
|
|
std::string trim_char_right(std::string s, char remove)
|
|
{
|
|
s.erase(std::find_if(s.rbegin(), s.rend(), [&](char c) { return c != remove; }).base(), s.end());
|
|
return s;
|
|
}
|
|
|
|
std::string trim_char(std::string s, char remove)
|
|
{
|
|
return trim_char_left(trim_char_right(std::move(s), remove), remove);
|
|
}
|
|
|
|
|
|
std::string escape_regex(const std::string& input)
|
|
{
|
|
std::string result;
|
|
result.reserve(input.length() * 2); // Reserve space for potential escapes
|
|
|
|
for (char c : input) {
|
|
// Escape special regex characters
|
|
if (std::string("\\^$.|?*+()[{}]").find(c) != std::string::npos) {
|
|
result += '\\';
|
|
}
|
|
result += c;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string string_replace(const std::string& source, const std::string& old_str, const std::string& new_str)
|
|
{
|
|
return std::regex_replace(source, std::regex(escape_regex(old_str)), new_str);
|
|
/*
|
|
std::string result { source };
|
|
auto pos = result.find(old_str);
|
|
std::string old_str_e = old_str; // escape_regex(old_str);
|
|
while (pos != std::string::npos) {
|
|
// result = result.replace(pos, old_str.size(), new_str);
|
|
result = result.replace(pos, old_str_e.size(), new_str);
|
|
// pos = result.find(old_str);
|
|
pos = result.find(old_str_e);
|
|
// std::cout << " " << result << "\n";
|
|
}
|
|
return result;
|
|
*/
|
|
}
|
|
|
|
bool contains(const std::string& str, const std::string& substr)
|
|
{
|
|
return str.find(substr) != std::string::npos;
|
|
}
|
|
|
|
bool contains(const std::vector<std::string>& strings, const std::string& element)
|
|
{
|
|
return std::find(strings.begin(), strings.end(), element) != strings.end();
|
|
}
|
|
|
|
std::string regex_escape(const std::string& s)
|
|
{
|
|
/*
|
|
regex special { R"([\$.|?*+(){})" }; // ^ is reserved
|
|
return regex_replace(s, special, "\\[&$]");
|
|
*/
|
|
std::set chars { '\\', '|', '(', ')', '{', '}', '[', ']', '$', '^' };
|
|
std::string result {};
|
|
for (char c : s) {
|
|
if (chars.find(c) != chars.end())
|
|
result += "\\";
|
|
result += c;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
strings_t regex_split(const std::string& s, const std::regex& re, bool trim_parts)
|
|
{
|
|
strings_t result = {};
|
|
if (s.empty()) {
|
|
return result;
|
|
}
|
|
auto it = std::sregex_token_iterator(s.begin(), s.end(), re, -1);
|
|
while (it != std::sregex_token_iterator()) {
|
|
std::string part { *it };
|
|
if (trim_parts)
|
|
part = trim(part);
|
|
result.push_back(part);
|
|
it++;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
strings_t word_split(const std::string& s)
|
|
{
|
|
return regex_split(s, std::regex("\\s+"));
|
|
}
|
|
|
|
std::string collapse_whitespace(const std::string& s)
|
|
{
|
|
std::string result {};
|
|
bool in_space = true; // leading whitespace is dropped
|
|
for (char c : s) {
|
|
if (std::isspace(static_cast<unsigned char>(c)) != 0) {
|
|
in_space = true;
|
|
} else {
|
|
if (in_space && !result.empty()) {
|
|
result += ' ';
|
|
}
|
|
in_space = false;
|
|
result += c;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool contains_fold(const std::string& haystack, const std::string& needle)
|
|
{
|
|
auto fold = [](const std::string& s) {
|
|
std::string result {};
|
|
for (char c : s) {
|
|
result += static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
|
|
}
|
|
return result;
|
|
};
|
|
return fold(haystack).find(fold(needle)) != std::string::npos;
|
|
}
|
|
|
|
bool is_in(const std::string& s, const strings_t& v)
|
|
{
|
|
return find(v.begin(), v.end(), s) != v.end();
|
|
}
|
|
|
|
bool is_not_in(const std::string& s, const strings_t& v)
|
|
{
|
|
return find(v.begin(), v.end(), s) == v.end();
|
|
}
|
|
|
|
strings_t find_all(const std::string& str, const std::regex& pattern, int match_group)
|
|
{
|
|
std::sregex_iterator end {};
|
|
strings_t result;
|
|
for (std::sregex_iterator p {str.begin(), str.end(), pattern}; p!= end; ++p)
|
|
result.push_back((*p)[match_group]);
|
|
return result;
|
|
}
|
|
|
|
strings_t find_all(const std::string& str, const std::string& pattern, int match_group)
|
|
{
|
|
return find_all(str, std::regex(pattern), match_group);
|
|
}
|
|
|
|
strings_t split_into_paragraphs(const std::string& s)
|
|
{
|
|
std::string t {trim(s)};
|
|
std::string marker { "_PAR_" };
|
|
t = trim(std::regex_replace(t, std::regex(R"(\n *(\n *)+)"), marker)) + marker;
|
|
//return find_all(t, regex(R"(((\s|.)*?)" + marker + ")"), 1);
|
|
return find_all(t, std::regex(R"(((\s|.)*?)_PAR_)"), 1);
|
|
|
|
}
|
|
|
|
std::string add_margin(const std::string& s, unsigned int margin_size)
|
|
{
|
|
auto margin = std::string(margin_size, ' ');
|
|
return trim_right(
|
|
margin + std::regex_replace(s, std::regex(R"(\n)"), '\n' + margin));
|
|
}
|
|
|
|
std::string justify_string(const std::string& s, unsigned int width=80, bool french_spacing=false)
|
|
{
|
|
strings_t words = find_all(s, R"([^\s]+)");
|
|
std::stringstream ss {};
|
|
std::stringstream line {};
|
|
for (std::string w : words) {
|
|
if (line.str().size() + w.size() + 1 > width) {
|
|
ss << line.str() << '\n';
|
|
line.str("");
|
|
}
|
|
if (not french_spacing and w[w.size()-1] == '.')
|
|
w += " ";
|
|
line << w << " ";
|
|
}
|
|
if (!line.str().empty())
|
|
ss << line.str();
|
|
std::string result = trim(ss.str());
|
|
result = std::regex_replace(result, std::regex("~"), " ");
|
|
return result;
|
|
}
|
|
|
|
// The error-message formatting contract (2026-08-21): a message carries no
|
|
// decisions about line breaks EXCEPT by indentation.
|
|
// * a line beginning with whitespace is VERBATIM -- an example, a pattern,
|
|
// a signature, a list entry -- emitted untouched: no folding, no
|
|
// wrapping, no "~" substitution (a pattern may contain a literal ~);
|
|
// * blank lines separate blocks (runs collapse to one);
|
|
// * everything else is prose: consecutive lines fold into one paragraph
|
|
// and are wrapped to the width.
|
|
// So prose stays machine-wrapped however the source hand-wraps it, and the
|
|
// one legitimate exception is marked in the one place it cannot be missed.
|
|
// This replaced a per-call do_justify flag on the Error constructors, whose
|
|
// decision lived apart from the text it governed. Line-based by hand: no
|
|
// std::regex over the whole message (the ambiguous-alternation hazard).
|
|
std::string justify(
|
|
const std::string& input_text, unsigned int text_width, unsigned int margin_width)
|
|
{
|
|
text_width -= margin_width;
|
|
strings_t lines {};
|
|
{
|
|
const std::string text = trim(input_text);
|
|
std::string::size_type from = 0;
|
|
while (from <= text.size()) {
|
|
auto nl = text.find('\n', from);
|
|
if (nl == std::string::npos) {
|
|
lines.push_back(text.substr(from));
|
|
break;
|
|
}
|
|
lines.push_back(text.substr(from, nl - from));
|
|
from = nl + 1;
|
|
}
|
|
}
|
|
std::string result {};
|
|
strings_t prose {};
|
|
bool pending_blank = false;
|
|
auto emit = [&](const std::string& block) {
|
|
if (!result.empty()) {
|
|
result += pending_blank ? "\n\n" : "\n";
|
|
}
|
|
result += block;
|
|
pending_blank = false;
|
|
};
|
|
auto flush_prose = [&]() {
|
|
if (!prose.empty()) {
|
|
emit(justify_string(join(prose, " "), text_width));
|
|
prose.clear();
|
|
}
|
|
};
|
|
for (const std::string& line : lines) {
|
|
if (trim(line).empty()) { // block separator
|
|
flush_prose();
|
|
pending_blank = !result.empty();
|
|
} else if (line[0] == ' ' || line[0] == '\t') { // verbatim line
|
|
flush_prose();
|
|
emit(trim_right(line));
|
|
} else { // prose
|
|
prose.push_back(trim(line));
|
|
}
|
|
}
|
|
flush_prose();
|
|
result += "\n";
|
|
if (margin_width > 0) {
|
|
result = add_margin(result, margin_width);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string join(const strings_t& ss, const std::string& separator)
|
|
{
|
|
if (ss.empty()) {
|
|
return std::string();
|
|
} else if (ss.size() == 1) {
|
|
return ss[0];
|
|
} else {
|
|
std::stringstream strm {};
|
|
std::copy(ss.begin(), ss.end() - 1,
|
|
std::ostream_iterator<std::string>(strm, separator.c_str()));
|
|
strm << ss.back();
|
|
return strm.str();
|
|
}
|
|
}
|
|
|
|
std::string join(int argc, char* argv[], const std::string& separator)
|
|
{
|
|
std::string result {};
|
|
for (int i = 0; i < argc; ++i) {
|
|
// Append directly (see argv_to_string): avoids a temporary and the same
|
|
// GCC 12 -Wrestrict false positive.
|
|
result += argv[i];
|
|
result += separator;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string argv_to_string(int argc, char* argv[])
|
|
{
|
|
if (argc == 0) {
|
|
return "";
|
|
}
|
|
std::string result = argv[0];
|
|
for (int i = 1; i < argc; ++i) {
|
|
// Append the pieces directly rather than building a `" " + string(...)`
|
|
// temporary: identical result, avoids an allocation, and sidesteps a
|
|
// GCC 12 -Wrestrict false positive on the temporary's memcpy.
|
|
result += ' ';
|
|
result += argv[i];
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
std::string plural(const std::string& word, int count)
|
|
{
|
|
std::string result {word};
|
|
if (count != 1) {
|
|
if (*(word.end()-1) == 'y')
|
|
result = word.substr(0, word.size()-2) + "ies";
|
|
else
|
|
result = word + "s";
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string plural(const std::string& word, const strings_t& things)
|
|
{
|
|
std::string result { word };
|
|
if (things.size() != 1) {
|
|
if (*(word.end()-1) == 'y')
|
|
result = word.substr(0, word.size()-2) + "ies";
|
|
else
|
|
result = word + "s";
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string to_be(int count, bool present)
|
|
{
|
|
std::string result {};
|
|
if (count > 1) {
|
|
if (present) {
|
|
result = "are";
|
|
} else {
|
|
result = "were";
|
|
}
|
|
} else {
|
|
if (present) {
|
|
result = "is";
|
|
} else {
|
|
result = "was";
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
int max_length(const strings_t& ss)
|
|
{
|
|
size_t result = 0;
|
|
for_each(ss.begin(), ss.end(),
|
|
[&result](const std::string& s) { result = std::max(result, s.size()); });
|
|
return result;
|
|
}
|
|
|
|
|
|
/*
|
|
std::vector<std::string> map_key_lengths(std::map<std::string, auto> map)
|
|
{
|
|
int result = 0;
|
|
for (auto const& item: map) {
|
|
result = std::max(result, item.first.size());
|
|
}
|
|
}
|
|
|
|
|
|
int
|
|
std::map<int, int> m;
|
|
std::vector<int> key, value;
|
|
for(std::map<int,int>::iterator it = m.begin(); it != m.end(); ++it) {
|
|
key.push_back(it->first);
|
|
value.push_back(it->second);
|
|
std::cout << "Key: " << it->first << std::endl;
|
|
std::cout << "Value: " << it->second << std::endl;
|
|
}
|
|
*/
|
|
|
|
|
|
std::vector<std::pair<std::string, std::string>> environment_variables(bool allow_empty_definitions)
|
|
{
|
|
// std::cout << "read_environment:\n";
|
|
std::vector<std::pair<std::string, std::string>> result {};
|
|
extern char **environ;
|
|
for (int i = 0; environ[i]; i++) {
|
|
auto parts = regex_split(environ[i], std::regex("="), true);
|
|
if (!allow_empty_definitions && parts.size() < 2) {
|
|
throw Internal_error(
|
|
"Incorrect environment variable format:\n " + std::string(environ[i]),
|
|
Locator());
|
|
}
|
|
std::string name = parts[0];
|
|
parts.erase(parts.begin());
|
|
std::string value = join(parts, "=");
|
|
// std::cout << name << sp_arrow << value << "\n";
|
|
result.push_back({name, value});
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string replace_environment_variables(const std::string& str)
|
|
{
|
|
if (str.find('{') == std::string::npos || str.find('}') == std::string::npos) {
|
|
return str;
|
|
}
|
|
if (str.find('{') == std::string::npos || str.find('\n') != std::string::npos) {
|
|
return str;
|
|
}
|
|
|
|
if (str.size() < 3) {
|
|
return str;
|
|
}
|
|
std::regex variable_re(R"((.*?)\{([A-Z_]+)\})");
|
|
std::sregex_iterator end {};
|
|
std::string result {};
|
|
//sregex_iterator q {};
|
|
size_t endpos = 0;
|
|
for (std::sregex_iterator p {str.begin(), str.end(), variable_re}; p!= end; ++p) {
|
|
std::smatch m = *p;
|
|
std::string prefix = m[1];
|
|
std::string var = m[2];
|
|
endpos = m.position() + m.length();
|
|
std::string envvar = get_env_var(var);
|
|
result += prefix + envvar;
|
|
}
|
|
if (endpos < str.size() - 1) {
|
|
result += str.substr(endpos);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string abbrev(const std::string& s, unsigned int max_length, bool remove_newlines)
|
|
{
|
|
std::string result {s};
|
|
if (s.size() > max_length) {
|
|
if (remove_newlines) {
|
|
result = trim(result);
|
|
result = std::regex_replace(result, std::regex("\n"), broken_bar);
|
|
}
|
|
int suffix_size = 8;
|
|
// Plain, deliberately. This string is spliced into another and shown
|
|
// later, on a stream this function cannot know -- so a colour baked in
|
|
// here could reach a pipe or a file. It also fixes an arithmetic
|
|
// error that the colours caused: ellipsis.size() counted the 14
|
|
// characters of the two escape sequences as visible width, so the
|
|
// abbreviation was cut 14 characters shorter than max_length asked.
|
|
std::string ellipsis { "[...]" };
|
|
int prefix_end = max_length - suffix_size - static_cast<int>(ellipsis.size());
|
|
result = result.replace(result.begin() + prefix_end,
|
|
result.end() - suffix_size,
|
|
ellipsis);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void remove_element(std::vector<std::string>& ss, const std::string& removed)
|
|
{
|
|
ss.erase(std::remove_if(ss.begin(), ss.end(),
|
|
[&removed](const std::string& s) { return s == removed; }),
|
|
ss.end());
|
|
}
|
|
|
|
void remove_duplicates(strings_t& ss)
|
|
{
|
|
// https://en.cppreference.com/w/cpp/algorithm/unique
|
|
std::sort(ss.begin(), ss.end());
|
|
auto last = std::unique(ss.begin(), ss.end());
|
|
ss.erase(last, ss.end());
|
|
}
|
|
|
|
std::string display_string(const std::string& s, unsigned int width, bool replace_newlines)
|
|
{
|
|
std::string suffix { "..." };
|
|
std::string result { s };
|
|
if (replace_newlines)
|
|
result = std::regex_replace(result, std::regex("\n"), "/");
|
|
auto rlen = result.length();
|
|
auto slen = suffix.length();
|
|
if ((rlen > slen) and (rlen - slen) > width) {
|
|
result = result.replace(result.begin()+width, result.end(), suffix); //substr(0, width) + suffix;
|
|
}
|
|
//result = '"' + result + '"';
|
|
return result;
|
|
}
|
|
|
|
std::pair<std::string,std::string> extract_parameter_type(const std::string& parameter_name)
|
|
{
|
|
std::regex name_pat { R"((\w+)\.(\w+))" };
|
|
std::smatch match {};
|
|
std::string type_name = "string";
|
|
std::string name = parameter_name;
|
|
if (std::regex_match(parameter_name, match, name_pat)) {
|
|
name = match[1];
|
|
type_name = match[2];
|
|
}
|
|
return { name, type_name };
|
|
}
|
|
|
|
std::tuple<std::string, std::string, bool> regex_split_prefix(const std::regex& pattern, const std::string& text)
|
|
{
|
|
std::smatch match;
|
|
if (std::regex_search(text, match, pattern) && match.position() == 0) {
|
|
// Match found at the beginning of the string
|
|
std::string matched = match.str();
|
|
std::string remainder = text.substr(matched.length());
|
|
return { matched, remainder, true };
|
|
} else {
|
|
// No match at the beginning
|
|
return { "", text, false };
|
|
}
|
|
}
|
|
|
|
std::vector<std::string> dlist_split(const std::string& s)
|
|
{
|
|
// If a [^\w] character surrounded by spaces exists in s, it is the delimiter.
|
|
// If not, the first space-delimited word is the delimiter.
|
|
if (s.find(" ") == std::string::npos) { // Only one element.
|
|
//std::vector<std::string> result {s};
|
|
//return result;
|
|
return {s};
|
|
} else {
|
|
std::smatch match{};
|
|
std::string elements_str{s};
|
|
std::string delimiter {};
|
|
if (std::regex_search(s, match, std::regex(R"(\s+([^\w])\s+)"))) {
|
|
delimiter = match[1];
|
|
} else {
|
|
std::string::const_iterator iter =
|
|
std::find_if(s.cbegin(), s.cend(), [](char c) { return c == ' '; });
|
|
delimiter = std::string(s.cbegin(), iter);
|
|
elements_str = std::string(iter, s.cend());
|
|
}
|
|
std::vector<std::string> elements {
|
|
regex_split(elements_str, std::regex(delimiter), true) };
|
|
return elements;
|
|
}
|
|
}
|
|
|
|
std::string get_env_var(const std::string& var) {
|
|
std::lock_guard<std::mutex> lock(env_mutex);
|
|
const char* val = getenv(var.c_str());
|
|
return val ? std::string(val) : "";
|
|
}
|
|
|
|
std::string freplace(const std::string& src, const std::regex& pattern,
|
|
const std::function<std::string(std::smatch)>& func)
|
|
{
|
|
std::string result {};
|
|
std::smatch match;
|
|
|
|
bool found = std::regex_search(src.begin(), src.end(), match, pattern);
|
|
auto pos = src.begin();
|
|
// int n = 0;
|
|
while (found) {
|
|
std::string part(pos, pos + match.position());
|
|
result += part;
|
|
result += func(match);
|
|
pos += match.position(0) + match.length(0);
|
|
found = std::regex_search(pos, src.end(), match, pattern);
|
|
// n++;
|
|
}
|
|
std::string part(pos, pos + match.position());
|
|
result += part;
|
|
// std::cout << "Found " << n << " matches\n";
|
|
return result;
|
|
}
|
|
|
|
std::string exec(const char* cmd)
|
|
{
|
|
std::array<char, 128> buffer;
|
|
std::string result;
|
|
FILE* pipe = popen(cmd, "r");
|
|
if (!pipe) throw std::runtime_error("popen() failed");
|
|
while (fgets(buffer.data(), buffer.size(), pipe) != nullptr) {
|
|
result += buffer.data();
|
|
}
|
|
pclose(pipe);
|
|
return result;
|
|
}
|