diff --git a/src/core/regex/CMakeLists.txt b/src/core/regex/CMakeLists.txt index 958144e9d..f10eec028 100644 --- a/src/core/regex/CMakeLists.txt +++ b/src/core/regex/CMakeLists.txt @@ -1,5 +1,5 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME regex - SOURCES regex.cc iregexp.h permissive.h) + SOURCES regex.cc ecma262.h ecma262_properties.h iregexp.h permissive.h) if(SOURCEMETA_CORE_INSTALL) sourcemeta_library_install(NAMESPACE sourcemeta PROJECT core NAME regex) diff --git a/src/core/regex/ecma262.h b/src/core/regex/ecma262.h new file mode 100644 index 000000000..d5994afdf --- /dev/null +++ b/src/core/regex/ecma262.h @@ -0,0 +1,1026 @@ +#ifndef SOURCEMETA_CORE_REGEX_ECMA262_H_ +#define SOURCEMETA_CORE_REGEX_ECMA262_H_ + +#include + +#include "ecma262_properties.h" + +#include // std::size_t +#include // std::optional +#include // std::string, std::u32string +#include // std::string_view, std::u32string_view +#include // std::unordered_set +#include // std::move +#include // std::vector + +namespace sourcemeta::core { + +namespace { + +// Every code point is a possible member of a pattern, including the null one, +// so running past the end is reported with a value outside the Unicode range +constexpr char32_t ECMA262_END{0xFFFFFFFF}; + +constexpr char32_t ECMA262_LAST_CODE_POINT{0x10FFFF}; +constexpr char32_t ECMA262_LEAD_SURROGATE_FIRST{0xD800}; +constexpr char32_t ECMA262_LEAD_SURROGATE_LAST{0xDBFF}; +constexpr char32_t ECMA262_TRAIL_SURROGATE_FIRST{0xDC00}; +constexpr char32_t ECMA262_TRAIL_SURROGATE_LAST{0xDFFF}; +constexpr char32_t ECMA262_ZERO_WIDTH_NON_JOINER{0x200C}; +constexpr char32_t ECMA262_ZERO_WIDTH_JOINER{0x200D}; + +constexpr std::u32string_view ECMA262_SYNTAX_CHARACTERS{U"^$\\.*+?()[]{}|"}; +constexpr std::u32string_view ECMA262_CLASS_SHORTHANDS{U"dDsSwW"}; +constexpr std::u32string_view ECMA262_MODIFIERS{U"ims"}; + +// The characters that set notation reserves for its own operators, and the +// ones it only accepts escaped so that future syntax stays available +constexpr std::u32string_view ECMA262_SET_SYNTAX_CHARACTERS{U"()[]{}/-\\|"}; +constexpr std::u32string_view ECMA262_SET_RESERVED_PUNCTUATORS{ + U"&-!#%,:;<=>@`~"}; +constexpr std::u32string_view ECMA262_SET_DOUBLED_PUNCTUATORS{ + U"&!#$%*+,.:;<=>?@^`~"}; + +// The outcome of reading one member of a character class, where a shorthand +// or a property stands for a set rather than for a single code point +struct Ecma262ClassAtom { + bool is_class; + char32_t value; +}; + +// The outcome of reading one operand of a set expression, which additionally +// tracks whether it can stand for something other than a single code point +struct Ecma262SetOperand { + bool is_character; + char32_t value; + bool may_contain_strings; + bool is_range; +}; + +inline auto is_set_syntax_character(const char32_t codepoint) -> bool { + return ECMA262_SET_SYNTAX_CHARACTERS.find(codepoint) != + std::u32string_view::npos; +} + +inline auto is_set_reserved_punctuator(const char32_t codepoint) -> bool { + return ECMA262_SET_RESERVED_PUNCTUATORS.find(codepoint) != + std::u32string_view::npos; +} + +inline auto is_ascii_letter(const char32_t codepoint) -> bool { + return (codepoint >= U'a' && codepoint <= U'z') || + (codepoint >= U'A' && codepoint <= U'Z'); +} + +inline auto is_decimal_digit(const char32_t codepoint) -> bool { + return codepoint >= U'0' && codepoint <= U'9'; +} + +inline auto is_hex_code_point(const char32_t codepoint) -> bool { + return is_decimal_digit(codepoint) || + (codepoint >= U'a' && codepoint <= U'f') || + (codepoint >= U'A' && codepoint <= U'F'); +} + +inline auto hex_code_point_value(const char32_t codepoint) -> char32_t { + if (is_decimal_digit(codepoint)) { + return codepoint - U'0'; + } + + return (codepoint | 0x20) - U'a' + 10; +} + +inline auto is_identifier_start(const char32_t codepoint) -> bool { + return is_id_start(codepoint) || codepoint == U'$' || codepoint == U'_'; +} + +inline auto is_identifier_part(const char32_t codepoint) -> bool { + return is_id_continue(codepoint) || codepoint == U'$' || + codepoint == ECMA262_ZERO_WIDTH_NON_JOINER || + codepoint == ECMA262_ZERO_WIDTH_JOINER; +} + +inline auto significant_digits(const std::u32string_view digits) + -> std::u32string_view { + const auto first{digits.find_first_not_of(U'0')}; + return first == std::u32string_view::npos ? std::u32string_view{} + : digits.substr(first); +} + +// The digits of a quantifier bound are unbounded in the grammar, so the two +// bounds are ordered as decimal strings rather than as machine integers +inline auto decimal_greater(const std::u32string_view left, + const std::u32string_view right) -> bool { + const auto left_value{significant_digits(left)}; + const auto right_value{significant_digits(right)}; + return left_value.size() == right_value.size() + ? left_value > right_value + : left_value.size() > right_value.size(); +} + +// Groups and classes both nest, and reading them is recursive, so a pattern +// that nests past this many levels is turned down rather than allowed to run +// the stack out on input that may not be trusted. No real pattern comes close, +// and engines impose a bound of their own for the same reason +constexpr std::size_t ECMA262_MAXIMUM_DEPTH{256}; + +// A recursive descent reader of the pattern grammar of ECMA-262, taken with +// the Unicode parameter and without the one for set notation. Constraints +// that look ahead of the point where they are written, such as a reference to +// a group that appears later, are collected and settled once the whole +// pattern has been read +struct Ecma262Reader { + std::u32string_view input{}; + bool unicode_sets{false}; + std::size_t depth{0}; + std::size_t position{0}; + std::size_t capture_count{0}; + // The names that the alternative being read has declared, at every level of + // alternation, as a name may only repeat across alternatives + std::vector> declared; + std::unordered_set names; + std::vector references; + std::u32string largest_reference; + + [[nodiscard]] auto exhausted() const -> bool { + return this->position >= this->input.size(); + } + + [[nodiscard]] auto peek(const std::size_t offset = 0) const -> char32_t { + return this->position + offset < this->input.size() + ? this->input[this->position + offset] + : ECMA262_END; + } + + auto consume(const char32_t codepoint) -> bool { + if (this->peek() == codepoint) { + this->position += 1; + return true; + } + + return false; + } + + auto read_digits() -> std::u32string_view { + const auto start{this->position}; + while (is_decimal_digit(this->peek())) { + this->position += 1; + } + + return this->input.substr(start, this->position - start); + } + + auto read_pattern() -> bool { + if (!this->read_disjunction() || !this->exhausted()) { + return false; + } + + for (const auto &reference : this->references) { + if (!this->names.contains(reference)) { + return false; + } + } + + if (!this->largest_reference.empty() && + decimal_greater(this->largest_reference, + to_decimal_string(this->capture_count))) { + return false; + } + + return true; + } + + static auto to_decimal_string(const std::size_t value) -> std::u32string { + if (value == 0) { + return U"0"; + } + + std::u32string result; + for (auto remaining = value; remaining > 0; remaining /= 10) { + result.insert(result.begin(), + static_cast(U'0' + (remaining % 10))); + } + + return result; + } + + auto read_disjunction() -> bool { + if (this->depth >= ECMA262_MAXIMUM_DEPTH) { + return false; + } + + this->depth += 1; + const auto result{this->read_disjunction_contents()}; + this->depth -= 1; + return result; + } + + auto read_disjunction_contents() -> bool { + this->declared.emplace_back(); + std::unordered_set gathered; + if (!this->read_alternative()) { + return false; + } + + while (this->consume(U'|')) { + gathered.merge(this->declared.back()); + this->declared.back().clear(); + if (!this->read_alternative()) { + return false; + } + } + + gathered.merge(this->declared.back()); + this->declared.pop_back(); + + // What every alternative declared belongs to the alternative that encloses + // them, where the same name may not already stand + if (this->declared.empty()) { + return true; + } + + for (const auto &name : gathered) { + if (!this->declared.back().insert(name).second) { + return false; + } + } + + return true; + } + + auto read_alternative() -> bool { + while (!this->exhausted() && this->peek() != U'|' && this->peek() != U')') { + if (!this->read_term()) { + return false; + } + } + + return true; + } + + auto read_term() -> bool { + const auto current{this->peek()}; + // An assertion is its own kind of term and takes no quantifier, so a + // quantifier written after one has nothing to repeat + if (current == U'^' || current == U'$') { + this->position += 1; + return true; + } + + if (current == U'\\' && (this->peek(1) == U'b' || this->peek(1) == U'B')) { + this->position += 2; + return true; + } + + if (current == U'(' && this->peek(1) == U'?') { + const auto third{this->peek(2)}; + if (third == U'=' || third == U'!') { + this->position += 3; + return this->read_disjunction() && this->consume(U')'); + } + + if (third == U'<' && (this->peek(3) == U'=' || this->peek(3) == U'!')) { + this->position += 4; + return this->read_disjunction() && this->consume(U')'); + } + } + + return this->read_atom() && this->read_quantifier(); + } + + auto read_quantifier() -> bool { + const auto current{this->peek()}; + if (current == U'*' || current == U'+' || current == U'?') { + this->position += 1; + } else if (current == U'{') { + if (!this->read_braced_quantifier()) { + return false; + } + } else { + return true; + } + + this->consume(U'?'); + return true; + } + + auto read_braced_quantifier() -> bool { + this->position += 1; + const auto minimum{this->read_digits()}; + if (minimum.empty()) { + return false; + } + + if (this->consume(U'}')) { + return true; + } + + if (!this->consume(U',')) { + return false; + } + + if (this->consume(U'}')) { + return true; + } + + const auto maximum{this->read_digits()}; + return !maximum.empty() && this->consume(U'}') && + !decimal_greater(minimum, maximum); + } + + auto read_atom() -> bool { + const auto current{this->peek()}; + if (current == U'.') { + this->position += 1; + return true; + } + + if (current == U'\\') { + this->position += 1; + return this->read_atom_escape(); + } + + if (current == U'[') { + return this->read_character_class(); + } + + if (current == U'(') { + return this->read_group(); + } + + if (current == ECMA262_END || + ECMA262_SYNTAX_CHARACTERS.find(current) != std::u32string_view::npos) { + return false; + } + + this->position += 1; + return true; + } + + auto read_group() -> bool { + this->position += 1; + if (this->peek() != U'?') { + this->capture_count += 1; + return this->read_disjunction() && this->consume(U')'); + } + + if (this->peek(1) == U'<') { + this->position += 1; + auto name{this->read_group_name()}; + if (!name.has_value()) { + return false; + } + + this->capture_count += 1; + if (!this->declared.back().insert(name.value()).second) { + return false; + } + + this->names.insert(std::move(name.value())); + return this->read_disjunction() && this->consume(U')'); + } + + return this->read_modifiers() && this->read_disjunction() && + this->consume(U')'); + } + + // A group that carries no name carries flags instead, and the plain + // non-capturing form is the one that turns none of them on or off + auto read_modifiers() -> bool { + this->position += 1; + const auto added{this->read_modifier_run()}; + std::u32string removed; + const bool subtracts{this->consume(U'-')}; + if (subtracts) { + removed = this->read_modifier_run(); + } + + if (!this->consume(U':')) { + return false; + } + + if (subtracts && added.empty() && removed.empty()) { + return false; + } + + return !has_repeated_modifier(added) && !has_repeated_modifier(removed) && + !shares_modifier(added, removed); + } + + auto read_modifier_run() -> std::u32string { + std::u32string result; + while (ECMA262_MODIFIERS.find(this->peek()) != std::u32string_view::npos) { + result += this->peek(); + this->position += 1; + } + + return result; + } + + static auto has_repeated_modifier(const std::u32string_view run) -> bool { + for (std::size_t left = 0; left < run.size(); ++left) { + for (auto right = left + 1; right < run.size(); ++right) { + if (run[left] == run[right]) { + return true; + } + } + } + + return false; + } + + static auto shares_modifier(const std::u32string_view left, + const std::u32string_view right) -> bool { + for (const auto codepoint : left) { + if (right.find(codepoint) != std::u32string_view::npos) { + return true; + } + } + + return false; + } + + auto read_group_name() -> std::optional { + if (!this->consume(U'<')) { + return std::nullopt; + } + + std::u32string name; + const auto first{this->read_identifier_code_point()}; + if (!first.has_value() || !is_identifier_start(first.value())) { + return std::nullopt; + } + + name += first.value(); + while (this->peek() != U'>') { + const auto next{this->read_identifier_code_point()}; + if (!next.has_value() || !is_identifier_part(next.value())) { + return std::nullopt; + } + + name += next.value(); + } + + this->position += 1; + return name; + } + + auto read_identifier_code_point() -> std::optional { + const auto current{this->peek()}; + if (current == ECMA262_END) { + return std::nullopt; + } + + if (current == U'\\') { + if (this->peek(1) != U'u') { + return std::nullopt; + } + + this->position += 1; + return this->read_unicode_escape(); + } + + this->position += 1; + return current; + } + + auto read_atom_escape() -> bool { + const auto current{this->peek()}; + if (current >= U'1' && current <= U'9') { + const auto digits{this->read_digits()}; + if (decimal_greater(digits, this->largest_reference)) { + this->largest_reference = digits; + } + + return true; + } + + if (current == U'k') { + this->position += 1; + auto name{this->read_group_name()}; + if (!name.has_value()) { + return false; + } + + this->references.push_back(std::move(name.value())); + return true; + } + + if (ECMA262_CLASS_SHORTHANDS.find(current) != std::u32string_view::npos) { + this->position += 1; + return true; + } + + if (current == U'p' || current == U'P') { + return this->read_unicode_property(current == U'P').has_value(); + } + + return this->read_character_escape().has_value(); + } + + // Reads a property expression and reports whether it may stand for + // something other than a single code point + auto read_unicode_property(const bool negated) -> std::optional { + this->position += 1; + if (!this->consume(U'{')) { + return std::nullopt; + } + + const auto first{this->read_property_token()}; + if (this->consume(U'=')) { + const auto value{this->read_property_token()}; + if (first.empty() || value.empty() || !this->consume(U'}') || + !is_listed_property(ECMA262_NON_BINARY_PROPERTIES, first)) { + return std::nullopt; + } + + const bool known{ + (first == "General_Category" || first == "gc") + ? is_listed_property(ECMA262_GENERAL_CATEGORY_VALUES, value) + : is_listed_property(ECMA262_SCRIPT_VALUES, value)}; + return known ? std::optional{false} : std::nullopt; + } + + if (first.empty() || !this->consume(U'}')) { + return std::nullopt; + } + + // A property that stands for a set of strings only resolves under set + // notation, and never where the property is negated + if (is_listed_property(ECMA262_BINARY_PROPERTIES_OF_STRINGS, first)) { + return (this->unicode_sets && !negated) ? std::optional{true} + : std::nullopt; + } + + if (is_listed_property(ECMA262_GENERAL_CATEGORY_VALUES, first) || + is_listed_property(ECMA262_BINARY_PROPERTIES, first)) { + return false; + } + + return std::nullopt; + } + + // Both halves of a property expression are spelled with ASCII alone, so the + // token is gathered as a narrow string ready to look up + auto read_property_token() -> std::string { + std::string result; + while (is_ascii_letter(this->peek()) || this->peek() == U'_' || + is_decimal_digit(this->peek())) { + result += static_cast(this->peek()); + this->position += 1; + } + + return result; + } + + auto read_character_escape() -> std::optional { + const auto current{this->peek()}; + switch (current) { + case U'f': + this->position += 1; + return 0x000C; + case U'n': + this->position += 1; + return 0x000A; + case U'r': + this->position += 1; + return 0x000D; + case U't': + this->position += 1; + return 0x0009; + case U'v': + this->position += 1; + return 0x000B; + default: + break; + } + + if (current == U'c') { + const auto letter{this->peek(1)}; + if (!is_ascii_letter(letter)) { + return std::nullopt; + } + + this->position += 2; + return letter % 32; + } + + if (current == U'0') { + if (is_decimal_digit(this->peek(1))) { + return std::nullopt; + } + + this->position += 1; + return 0x0000; + } + + if (current == U'x') { + if (!is_hex_code_point(this->peek(1)) || + !is_hex_code_point(this->peek(2))) { + return std::nullopt; + } + + const auto value{(hex_code_point_value(this->peek(1)) << 4U) | + hex_code_point_value(this->peek(2))}; + this->position += 3; + return value; + } + + if (current == U'u') { + return this->read_unicode_escape(); + } + + if (current == U'/' || + ECMA262_SYNTAX_CHARACTERS.find(current) != std::u32string_view::npos) { + this->position += 1; + return current; + } + + return std::nullopt; + } + + auto read_unicode_escape() -> std::optional { + this->position += 1; + if (this->consume(U'{')) { + if (!is_hex_code_point(this->peek())) { + return std::nullopt; + } + + char32_t value{0}; + while (is_hex_code_point(this->peek())) { + value = (value << 4U) | hex_code_point_value(this->peek()); + this->position += 1; + if (value > ECMA262_LAST_CODE_POINT) { + return std::nullopt; + } + } + + return this->consume(U'}') ? std::optional{value} + : std::nullopt; + } + + const auto leading{this->read_four_hex_digits()}; + if (!leading.has_value()) { + return std::nullopt; + } + + // A pair of escapes standing for a surrogate pair denotes the single code + // point they combine into, rather than either half on its own + if (leading.value() >= ECMA262_LEAD_SURROGATE_FIRST && + leading.value() <= ECMA262_LEAD_SURROGATE_LAST && + this->peek() == U'\\' && this->peek(1) == U'u') { + const auto rewind{this->position}; + this->position += 2; + const auto trailing{this->read_four_hex_digits()}; + if (trailing.has_value() && + trailing.value() >= ECMA262_TRAIL_SURROGATE_FIRST && + trailing.value() <= ECMA262_TRAIL_SURROGATE_LAST) { + return 0x10000 + + ((leading.value() - ECMA262_LEAD_SURROGATE_FIRST) << 10U) + + (trailing.value() - ECMA262_TRAIL_SURROGATE_FIRST); + } + + this->position = rewind; + } + + return leading; + } + + auto read_four_hex_digits() -> std::optional { + for (std::size_t offset = 0; offset < 4; ++offset) { + if (!is_hex_code_point(this->peek(offset))) { + return std::nullopt; + } + } + + char32_t value{0}; + for (std::size_t offset = 0; offset < 4; ++offset) { + value = (value << 4U) | hex_code_point_value(this->peek(offset)); + } + + this->position += 4; + return value; + } + + auto read_character_class() -> bool { + this->position += 1; + const bool negated{this->consume(U'^')}; + if (this->unicode_sets) { + const auto strings{this->read_set_expression()}; + // A negated class can never stand for a set of strings, as there is no + // sensible complement of one + return strings.has_value() && !(negated && strings.value()) && + this->consume(U']'); + } + + while (this->peek() != U']') { + const auto first{this->read_class_atom()}; + if (!first.has_value()) { + return false; + } + + if (this->peek() != U'-' || this->peek(1) == U']' || + this->peek(1) == ECMA262_END) { + continue; + } + + this->position += 1; + const auto second{this->read_class_atom()}; + if (!second.has_value() || first->is_class || second->is_class || + first->value > second->value) { + return false; + } + } + + this->position += 1; + return true; + } + + // Reads the contents of a class written with set notation, reporting + // whether the result may stand for something other than a single code point + auto read_set_expression() -> std::optional { + if (this->depth >= ECMA262_MAXIMUM_DEPTH) { + return std::nullopt; + } + + this->depth += 1; + const auto result{this->read_set_expression_contents()}; + this->depth -= 1; + return result; + } + + auto read_set_expression_contents() -> std::optional { + if (this->peek() == U']') { + return false; + } + + auto first{this->read_set_range_or_operand()}; + if (!first.has_value()) { + return std::nullopt; + } + + const bool operates{(this->peek() == U'&' && this->peek(1) == U'&') || + (this->peek() == U'-' && this->peek(1) == U'-')}; + if (first->is_range && operates) { + return std::nullopt; + } + + if (this->peek() == U'&' && this->peek(1) == U'&') { + auto strings{first->may_contain_strings}; + while (this->peek() == U'&' && this->peek(1) == U'&') { + this->position += 2; + if (this->peek() == U'&') { + return std::nullopt; + } + + const auto operand{this->read_set_operand()}; + if (!operand.has_value()) { + return std::nullopt; + } + + strings = strings && operand->may_contain_strings; + } + + return this->peek() == U']' ? std::optional{strings} : std::nullopt; + } + + if (this->peek() == U'-' && this->peek(1) == U'-') { + while (this->peek() == U'-' && this->peek(1) == U'-') { + this->position += 2; + if (!this->read_set_operand().has_value()) { + return std::nullopt; + } + } + + // Taking members away can never introduce one, so only what the + // expression started from decides this + return this->peek() == U']' + ? std::optional{first->may_contain_strings} + : std::nullopt; + } + + auto strings{first->may_contain_strings}; + while (this->peek() != U']') { + const auto next{this->read_set_range_or_operand()}; + if (!next.has_value()) { + return std::nullopt; + } + + strings = strings || next->may_contain_strings; + } + + return strings; + } + + auto read_set_range_or_operand() -> std::optional { + const auto first{this->read_set_operand()}; + if (!first.has_value() || !first->is_character || this->peek() != U'-' || + this->peek(1) == U'-' || this->peek(1) == U']' || + this->peek(1) == ECMA262_END) { + return first; + } + + this->position += 1; + const auto second{this->read_set_operand()}; + if (!second.has_value() || !second->is_character || + first->value > second->value) { + return std::nullopt; + } + + return Ecma262SetOperand{.is_character = false, + .value = 0, + .may_contain_strings = false, + .is_range = true}; + } + + auto read_set_operand() -> std::optional { + if (this->peek() == U'[') { + this->position += 1; + const bool negated{this->consume(U'^')}; + const auto strings{this->read_set_expression()}; + if (!strings.has_value() || (negated && strings.value()) || + !this->consume(U']')) { + return std::nullopt; + } + + return Ecma262SetOperand{.is_character = false, + .value = 0, + .may_contain_strings = strings.value(), + .is_range = false}; + } + + if (this->peek() == U'\\') { + const auto next{this->peek(1)}; + if (next == U'q') { + this->position += 2; + return this->read_set_string_disjunction(); + } + + if (ECMA262_CLASS_SHORTHANDS.find(next) != std::u32string_view::npos) { + this->position += 2; + return Ecma262SetOperand{.is_character = false, + .value = 0, + .may_contain_strings = false, + .is_range = false}; + } + + if (next == U'p' || next == U'P') { + this->position += 1; + const auto strings{this->read_unicode_property(next == U'P')}; + if (!strings.has_value()) { + return std::nullopt; + } + + return Ecma262SetOperand{.is_character = false, + .value = 0, + .may_contain_strings = strings.value(), + .is_range = false}; + } + } + + const auto character{this->read_set_character()}; + if (!character.has_value()) { + return std::nullopt; + } + + return Ecma262SetOperand{.is_character = true, + .value = character.value(), + .may_contain_strings = false, + .is_range = false}; + } + + auto read_set_string_disjunction() -> std::optional { + if (!this->consume(U'{')) { + return std::nullopt; + } + + bool strings{false}; + while (true) { + std::size_t length{0}; + while (this->peek() != U'}' && this->peek() != U'|') { + if (!this->read_set_character().has_value()) { + return std::nullopt; + } + + length += 1; + } + + // An alternative of any length other than one stands for a string + strings = strings || length != 1; + if (!this->consume(U'|')) { + break; + } + } + + if (!this->consume(U'}')) { + return std::nullopt; + } + + return Ecma262SetOperand{.is_character = false, + .value = 0, + .may_contain_strings = strings, + .is_range = false}; + } + + auto read_set_character() -> std::optional { + const auto current{this->peek()}; + if (current == ECMA262_END) { + return std::nullopt; + } + + if (current == U'\\') { + this->position += 1; + if (this->peek() == U'b') { + this->position += 1; + return 0x0008; + } + + if (is_set_reserved_punctuator(this->peek())) { + const auto punctuator{this->peek()}; + this->position += 1; + return punctuator; + } + + return this->read_character_escape(); + } + + // Set notation keeps its own operators, and holds back every doubled + // punctuator so that later revisions may give them a meaning + if (is_set_syntax_character(current) || + (this->peek(1) == current && + ECMA262_SET_DOUBLED_PUNCTUATORS.find(current) != + std::u32string_view::npos)) { + return std::nullopt; + } + + this->position += 1; + return current; + } + + auto read_class_atom() -> std::optional { + const auto current{this->peek()}; + if (current == ECMA262_END) { + return std::nullopt; + } + + if (current != U'\\') { + this->position += 1; + return Ecma262ClassAtom{.is_class = false, .value = current}; + } + + this->position += 1; + const auto next{this->peek()}; + if (next == U'b') { + this->position += 1; + return Ecma262ClassAtom{.is_class = false, .value = 0x0008}; + } + + if (next == U'-') { + this->position += 1; + return Ecma262ClassAtom{.is_class = false, .value = U'-'}; + } + + if (ECMA262_CLASS_SHORTHANDS.find(next) != std::u32string_view::npos) { + this->position += 1; + return Ecma262ClassAtom{.is_class = true, .value = 0}; + } + + if (next == U'p' || next == U'P') { + return this->read_unicode_property(next == U'P').has_value() + ? std::optional{Ecma262ClassAtom{ + .is_class = true, .value = 0}} + : std::nullopt; + } + + const auto value{this->read_character_escape()}; + return value.has_value() ? std::optional{Ecma262ClassAtom{ + .is_class = false, .value = value.value()}} + : std::nullopt; + } +}; + +// Whether the given text is a pattern of ECMA-262 read with the Unicode +// parameter, which is the reading that a JSON Schema regular expression takes +inline auto is_ecma262_pattern(const std::string_view pattern) -> bool { + const auto decoded{utf8_to_utf32(pattern)}; + if (!decoded.has_value()) { + return false; + } + + Ecma262Reader unicode{}; + unicode.input = decoded.value(); + if (unicode.read_pattern()) { + return true; + } + + Ecma262Reader unicode_sets{}; + unicode_sets.input = decoded.value(); + unicode_sets.unicode_sets = true; + return unicode_sets.read_pattern(); +} + +} // namespace + +} // namespace sourcemeta::core + +#endif diff --git a/src/core/regex/ecma262_properties.h b/src/core/regex/ecma262_properties.h new file mode 100644 index 000000000..0b15d6962 --- /dev/null +++ b/src/core/regex/ecma262_properties.h @@ -0,0 +1,572 @@ +#ifndef SOURCEMETA_CORE_REGEX_ECMA262_PROPERTIES_H_ +#define SOURCEMETA_CORE_REGEX_ECMA262_PROPERTIES_H_ + +#include // std::ranges::binary_search +#include // std::array +#include // std::size_t +#include // std::string_view + +namespace sourcemeta::core { + +namespace { + +// The property names and aliases that may precede an equals sign, as listed +// by the non-binary property table of ECMA-262 +constexpr std::array ECMA262_NON_BINARY_PROPERTIES{ + {"General_Category", "Script", "Script_Extensions", "gc", "sc", "scx"}}; + +// The binary property names and aliases of the ECMA-262 table of the same +// name, which implementations may not extend +constexpr std::array ECMA262_BINARY_PROPERTIES{ + {"AHex", + "ASCII", + "ASCII_Hex_Digit", + "Alpha", + "Alphabetic", + "Any", + "Assigned", + "Bidi_C", + "Bidi_Control", + "Bidi_M", + "Bidi_Mirrored", + "CI", + "CWCF", + "CWCM", + "CWKCF", + "CWL", + "CWT", + "CWU", + "Case_Ignorable", + "Cased", + "Changes_When_Casefolded", + "Changes_When_Casemapped", + "Changes_When_Lowercased", + "Changes_When_NFKC_Casefolded", + "Changes_When_Titlecased", + "Changes_When_Uppercased", + "DI", + "Dash", + "Default_Ignorable_Code_Point", + "Dep", + "Deprecated", + "Dia", + "Diacritic", + "EBase", + "EComp", + "EMod", + "EPres", + "Emoji", + "Emoji_Component", + "Emoji_Modifier", + "Emoji_Modifier_Base", + "Emoji_Presentation", + "Ext", + "ExtPict", + "Extended_Pictographic", + "Extender", + "Gr_Base", + "Gr_Ext", + "Grapheme_Base", + "Grapheme_Extend", + "Hex", + "Hex_Digit", + "IDC", + "IDS", + "IDSB", + "IDST", + "IDS_Binary_Operator", + "IDS_Trinary_Operator", + "ID_Continue", + "ID_Start", + "Ideo", + "Ideographic", + "Join_C", + "Join_Control", + "LOE", + "Logical_Order_Exception", + "Lower", + "Lowercase", + "Math", + "NChar", + "Noncharacter_Code_Point", + "Pat_Syn", + "Pat_WS", + "Pattern_Syntax", + "Pattern_White_Space", + "QMark", + "Quotation_Mark", + "RI", + "Radical", + "Regional_Indicator", + "SD", + "STerm", + "Sentence_Terminal", + "Soft_Dotted", + "Term", + "Terminal_Punctuation", + "UIdeo", + "Unified_Ideograph", + "Upper", + "Uppercase", + "VS", + "Variation_Selector", + "White_Space", + "XIDC", + "XIDS", + "XID_Continue", + "XID_Start", + "space"}}; + +// The properties that only resolve where a pattern carries the parameter for +// the set notation, which this dialect never does +constexpr std::array ECMA262_BINARY_PROPERTIES_OF_STRINGS{ + {"Basic_Emoji", "Emoji_Keycap_Sequence", "RGI_Emoji", + "RGI_Emoji_Flag_Sequence", "RGI_Emoji_Modifier_Sequence", + "RGI_Emoji_Tag_Sequence", "RGI_Emoji_ZWJ_Sequence"}}; + +// The general category values and value aliases of the Unicode Character +// Database that this project vendors +constexpr std::array ECMA262_GENERAL_CATEGORY_VALUES{ + {"C", + "Cased_Letter", + "Cc", + "Cf", + "Close_Punctuation", + "Cn", + "Co", + "Combining_Mark", + "Connector_Punctuation", + "Control", + "Cs", + "Currency_Symbol", + "Dash_Punctuation", + "Decimal_Number", + "Enclosing_Mark", + "Final_Punctuation", + "Format", + "Initial_Punctuation", + "L", + "LC", + "Letter", + "Letter_Number", + "Line_Separator", + "Ll", + "Lm", + "Lo", + "Lowercase_Letter", + "Lt", + "Lu", + "M", + "Mark", + "Math_Symbol", + "Mc", + "Me", + "Mn", + "Modifier_Letter", + "Modifier_Symbol", + "N", + "Nd", + "Nl", + "No", + "Nonspacing_Mark", + "Number", + "Open_Punctuation", + "Other", + "Other_Letter", + "Other_Number", + "Other_Punctuation", + "Other_Symbol", + "P", + "Paragraph_Separator", + "Pc", + "Pd", + "Pe", + "Pf", + "Pi", + "Po", + "Private_Use", + "Ps", + "Punctuation", + "S", + "Sc", + "Separator", + "Sk", + "Sm", + "So", + "Space_Separator", + "Spacing_Mark", + "Surrogate", + "Symbol", + "Titlecase_Letter", + "Unassigned", + "Uppercase_Letter", + "Z", + "Zl", + "Zp", + "Zs", + "cntrl", + "digit", + "punct"}}; + +// The script values and value aliases of the same database, shared by both +// the script property and its extension counterpart +constexpr std::array ECMA262_SCRIPT_VALUES{ + {"Adlam", + "Adlm", + "Aghb", + "Ahom", + "Anatolian_Hieroglyphs", + "Arab", + "Arabic", + "Armenian", + "Armi", + "Armn", + "Avestan", + "Avst", + "Bali", + "Balinese", + "Bamu", + "Bamum", + "Bass", + "Bassa_Vah", + "Batak", + "Batk", + "Beng", + "Bengali", + "Berf", + "Beria_Erfe", + "Bhaiksuki", + "Bhks", + "Bopo", + "Bopomofo", + "Brah", + "Brahmi", + "Brai", + "Braille", + "Bugi", + "Buginese", + "Buhd", + "Buhid", + "Cakm", + "Canadian_Aboriginal", + "Cans", + "Cari", + "Carian", + "Caucasian_Albanian", + "Chakma", + "Cham", + "Cher", + "Cherokee", + "Chorasmian", + "Chrs", + "Common", + "Copt", + "Coptic", + "Cpmn", + "Cprt", + "Cuneiform", + "Cypriot", + "Cypro_Minoan", + "Cyrillic", + "Cyrl", + "Deseret", + "Deva", + "Devanagari", + "Diak", + "Dives_Akuru", + "Dogr", + "Dogra", + "Dsrt", + "Dupl", + "Duployan", + "Egyp", + "Egyptian_Hieroglyphs", + "Elba", + "Elbasan", + "Elym", + "Elymaic", + "Ethi", + "Ethiopic", + "Gara", + "Garay", + "Geor", + "Georgian", + "Glag", + "Glagolitic", + "Gong", + "Gonm", + "Goth", + "Gothic", + "Gran", + "Grantha", + "Greek", + "Grek", + "Gujarati", + "Gujr", + "Gukh", + "Gunjala_Gondi", + "Gurmukhi", + "Guru", + "Gurung_Khema", + "Han", + "Hang", + "Hangul", + "Hani", + "Hanifi_Rohingya", + "Hano", + "Hanunoo", + "Hatr", + "Hatran", + "Hebr", + "Hebrew", + "Hira", + "Hiragana", + "Hluw", + "Hmng", + "Hmnp", + "Hrkt", + "Hung", + "Imperial_Aramaic", + "Inherited", + "Inscriptional_Pahlavi", + "Inscriptional_Parthian", + "Ital", + "Java", + "Javanese", + "Kaithi", + "Kali", + "Kana", + "Kannada", + "Katakana", + "Katakana_Or_Hiragana", + "Kawi", + "Kayah_Li", + "Khar", + "Kharoshthi", + "Khitan_Small_Script", + "Khmer", + "Khmr", + "Khoj", + "Khojki", + "Khudawadi", + "Kirat_Rai", + "Kits", + "Knda", + "Krai", + "Kthi", + "Lana", + "Lao", + "Laoo", + "Latin", + "Latn", + "Lepc", + "Lepcha", + "Limb", + "Limbu", + "Lina", + "Linb", + "Linear_A", + "Linear_B", + "Lisu", + "Lyci", + "Lycian", + "Lydi", + "Lydian", + "Mahajani", + "Mahj", + "Maka", + "Makasar", + "Malayalam", + "Mand", + "Mandaic", + "Mani", + "Manichaean", + "Marc", + "Marchen", + "Masaram_Gondi", + "Medefaidrin", + "Medf", + "Meetei_Mayek", + "Mend", + "Mende_Kikakui", + "Merc", + "Mero", + "Meroitic_Cursive", + "Meroitic_Hieroglyphs", + "Miao", + "Mlym", + "Modi", + "Mong", + "Mongolian", + "Mro", + "Mroo", + "Mtei", + "Mult", + "Multani", + "Myanmar", + "Mymr", + "Nabataean", + "Nag_Mundari", + "Nagm", + "Nand", + "Nandinagari", + "Narb", + "Nbat", + "New_Tai_Lue", + "Newa", + "Nko", + "Nkoo", + "Nshu", + "Nushu", + "Nyiakeng_Puachue_Hmong", + "Ogam", + "Ogham", + "Ol_Chiki", + "Ol_Onal", + "Olck", + "Old_Hungarian", + "Old_Italic", + "Old_North_Arabian", + "Old_Permic", + "Old_Persian", + "Old_Sogdian", + "Old_South_Arabian", + "Old_Turkic", + "Old_Uyghur", + "Onao", + "Oriya", + "Orkh", + "Orya", + "Osage", + "Osge", + "Osma", + "Osmanya", + "Ougr", + "Pahawh_Hmong", + "Palm", + "Palmyrene", + "Pau_Cin_Hau", + "Pauc", + "Perm", + "Phag", + "Phags_Pa", + "Phli", + "Phlp", + "Phnx", + "Phoenician", + "Plrd", + "Prti", + "Psalter_Pahlavi", + "Qaac", + "Qaai", + "Rejang", + "Rjng", + "Rohg", + "Runic", + "Runr", + "Samaritan", + "Samr", + "Sarb", + "Saur", + "Saurashtra", + "Sgnw", + "Sharada", + "Shavian", + "Shaw", + "Shrd", + "Sidd", + "Siddham", + "Sidetic", + "Sidt", + "SignWriting", + "Sind", + "Sinh", + "Sinhala", + "Sogd", + "Sogdian", + "Sogo", + "Sora", + "Sora_Sompeng", + "Soyo", + "Soyombo", + "Sund", + "Sundanese", + "Sunu", + "Sunuwar", + "Sylo", + "Syloti_Nagri", + "Syrc", + "Syriac", + "Tagalog", + "Tagb", + "Tagbanwa", + "Tai_Le", + "Tai_Tham", + "Tai_Viet", + "Tai_Yo", + "Takr", + "Takri", + "Tale", + "Talu", + "Tamil", + "Taml", + "Tang", + "Tangsa", + "Tangut", + "Tavt", + "Tayo", + "Telu", + "Telugu", + "Tfng", + "Tglg", + "Thaa", + "Thaana", + "Thai", + "Tibetan", + "Tibt", + "Tifinagh", + "Tirh", + "Tirhuta", + "Tnsa", + "Todhri", + "Todr", + "Tolong_Siki", + "Tols", + "Toto", + "Tulu_Tigalari", + "Tutg", + "Ugar", + "Ugaritic", + "Unknown", + "Vai", + "Vaii", + "Vith", + "Vithkuqi", + "Wancho", + "Wara", + "Warang_Citi", + "Wcho", + "Xpeo", + "Xsux", + "Yezi", + "Yezidi", + "Yi", + "Yiii", + "Zanabazar_Square", + "Zanb", + "Zinh", + "Zyyy", + "Zzzz"}}; + +// Every table above is sorted so that a lookup does not walk it end to end +template +auto is_listed_property(const std::array &table, + const std::string_view name) -> bool { + return std::ranges::binary_search(table, name); +} + +} // namespace + +} // namespace sourcemeta::core + +#endif diff --git a/src/core/regex/include/sourcemeta/core/regex.h b/src/core/regex/include/sourcemeta/core/regex.h index 9fd243066..4b0de5570 100644 --- a/src/core/regex/include/sourcemeta/core/regex.h +++ b/src/core/regex/include/sourcemeta/core/regex.h @@ -172,17 +172,45 @@ auto replace_all(const Regex ®ex, const std::string_view subject, /// @ingroup regex /// -/// Check whether the given string is a valid ECMA-262 regular expression. For -/// example: +/// Check whether the given string is a valid ECMA-262 regular expression. +/// +/// The pattern is read against the grammar of ECMA-262, including its early +/// errors, rather than handed to the underlying engine, so the answer follows +/// the standard instead of whatever a particular engine happens to accept. A +/// pattern is accepted when it parses under either of the two Unicode-aware +/// readings of the standard, meaning the one that JavaScript selects with the +/// `u` flag or the one it selects with the `v` flag. The latter is what makes +/// set notation, such as nested classes, set operations and string +/// disjunctions, come out valid. +/// +/// The legacy reading of Annex B, which JavaScript selects when no flag is +/// given, is deliberately not accepted. That reading treats a great deal of +/// otherwise malformed input as literal text, so a pattern written for another +/// flavour would pass without meaning what its author intended. The official +/// JSON Schema test suite agrees, as it requires an alarm escape to be +/// rejected even though the legacy reading accepts it as a literal. +/// +/// In practice this means that a syntax character standing on its own, such as +/// an unescaped brace or closing bracket, makes a pattern invalid, and that +/// property escapes only resolve against the property names and values that +/// the standard permits. For example: /// /// ```cpp /// #include /// #include /// /// assert(sourcemeta::core::is_regex_ecma("([abc])+\\s+$")); +/// assert(sourcemeta::core::is_regex_ecma("\\p{gc=Lu}")); +/// assert(sourcemeta::core::is_regex_ecma("[[a-z]--[aeiou]]")); /// assert(!sourcemeta::core::is_regex_ecma("^(abc]")); /// assert(!sourcemeta::core::is_regex_ecma("\\a")); +/// assert(!sourcemeta::core::is_regex_ecma("^{.*}$")); /// ``` +/// +/// Note that a pattern being valid does not mean this project can compile it, +/// as the standard places no bound on how much a quantifier may repeat while +/// the underlying engine does. Use `to_regex` to find out whether a pattern +/// can also be matched with. SOURCEMETA_CORE_REGEX_EXPORT auto is_regex_ecma(const std::string_view pattern) -> bool; diff --git a/src/core/regex/permissive.h b/src/core/regex/permissive.h index 8273beeba..4b395298c 100644 --- a/src/core/regex/permissive.h +++ b/src/core/regex/permissive.h @@ -15,7 +15,7 @@ namespace sourcemeta::core { namespace { -constexpr std::array, 42> +constexpr std::array, 47> UNICODE_PROPERTY_MAP{{{"digit", "Nd"}, {"Decimal_Number", "Nd"}, {"space", "White_Space"}, @@ -24,18 +24,21 @@ constexpr std::array, 42> {"Hex_Digit", "Hex_Digit"}, {"Alphabetic", "Alphabetic"}, {"Letter", "L"}, + {"Cased_Letter", "LC"}, {"Uppercase_Letter", "Lu"}, {"Lowercase_Letter", "Ll"}, {"Titlecase_Letter", "Lt"}, {"Modifier_Letter", "Lm"}, {"Other_Letter", "Lo"}, {"Mark", "M"}, + {"Combining_Mark", "M"}, {"Nonspacing_Mark", "Mn"}, {"Spacing_Mark", "Mc"}, {"Enclosing_Mark", "Me"}, {"Number", "N"}, {"Letter_Number", "Nl"}, {"Other_Number", "No"}, + {"punct", "P"}, {"Punctuation", "P"}, {"Connector_Punctuation", "Pc"}, {"Dash_Punctuation", "Pd"}, @@ -54,10 +57,15 @@ constexpr std::array, 42> {"Line_Separator", "Zl"}, {"Paragraph_Separator", "Zp"}, {"Other", "C"}, + {"cntrl", "Cc"}, {"Control", "Cc"}, {"Format", "Cf"}, {"Unassigned", "Cn"}, - {"Private_Use", "Co"}}}; + {"Private_Use", "Co"}, + {"Surrogate", "Cs"}}}; + +constexpr std::array GENERAL_CATEGORY_PREFIXES{ + {"General_Category=", "gc="}}; constexpr std::string_view SHORTHAND_CHARS{"dDwWsS"}; constexpr std::string_view SIMPLE_ESCAPES{"btnrfv0"}; @@ -577,15 +585,37 @@ inline auto expand_char_class(const std::string &content) return result.none() ? "(?!)" : bitset_to_class(result); } +inline auto property_class(const std::string_view name, const bool negated) + -> std::string { + return std::string("\\") + (negated ? 'P' : 'p') + '{' + std::string{name} + + '}'; +} + inline auto translate_property(const std::string_view name, const bool negated) -> std::optional { + // ECMA-262 resolves a lone property value against the general category + // before anything else, so naming that property outright means the same as + // giving the value on its own + auto value{name}; + bool general_category{false}; + for (const auto prefix : GENERAL_CATEGORY_PREFIXES) { + if (value.starts_with(prefix)) { + value.remove_prefix(prefix.size()); + general_category = true; + break; + } + } + for (const auto &[prop_name, pcre_name] : UNICODE_PROPERTY_MAP) { - if (name == prop_name) { - return std::string("\\") + (negated ? 'P' : 'p') + '{' + - std::string(pcre_name) + '}'; + if (value == prop_name) { + return property_class(pcre_name, negated); } } + if (general_category) { + return property_class(value, negated); + } + return std::nullopt; } diff --git a/src/core/regex/regex.cc b/src/core/regex/regex.cc index c5b505def..eb1d3b0dd 100644 --- a/src/core/regex/regex.cc +++ b/src/core/regex/regex.cc @@ -3,6 +3,7 @@ #include +#include "ecma262.h" #include "iregexp.h" #include "permissive.h" @@ -37,60 +38,6 @@ auto compile_pcre2(const std::string &pattern, const std::uint32_t options) return RegexTypePCRE2{std::shared_ptr(pcre2_regex)}; } -auto compiles_with_ecma_options(const std::string &pattern) -> bool { - int pcre2_error_code{0}; - PCRE2_SIZE pcre2_error_offset{0}; - pcre2_code *pcre2_regex_raw{pcre2_compile( - reinterpret_cast(pattern.c_str()), pattern.size(), - // Capturing groups are kept enabled so that ECMA-262 numbered - // backreferences like (a)\1 compile and match, at a small tracking cost - PCRE2_UTF | PCRE2_UCP | PCRE2_DOTALL | PCRE2_DOLLAR_ENDONLY | - PCRE2_NEVER_BACKSLASH_C | PCRE2_MATCH_INVALID_UTF | - PCRE2_ALLOW_EMPTY_CLASS, - &pcre2_error_code, &pcre2_error_offset, nullptr)}; - - if (pcre2_regex_raw == nullptr) { - return false; - } - - pcre2_code_free(pcre2_regex_raw); - return true; -} - -auto lookbehinds_as_lookaheads(const std::string &pattern) -> std::string { - std::string result; - result.reserve(pattern.size()); - bool in_class{false}; - for (std::size_t position = 0; position < pattern.size(); ++position) { - const char current{pattern[position]}; - if (current == '\\' && position + 1 < pattern.size()) { - result += current; - result += pattern[position + 1]; - position += 1; - continue; - } - - if (current == '[') { - in_class = true; - } else if (current == ']') { - in_class = false; - } - - if (!in_class && current == '(' && position + 3 < pattern.size() && - pattern[position + 1] == '?' && pattern[position + 2] == '<' && - (pattern[position + 3] == '=' || pattern[position + 3] == '!')) { - result += "(?"; - result += pattern[position + 3]; - position += 3; - continue; - } - - result += current; - } - - return result; -} - } // namespace auto to_regex(const std::string_view pattern, const RegexDialect dialect, @@ -266,23 +213,7 @@ auto matches_if_valid(const std::string_view pattern, } auto is_regex_ecma(const std::string_view pattern) -> bool { - const auto translated{translate_permissive(std::string{pattern})}; - if (!translated.ecma_valid || !translated.transformed.has_value()) { - return false; - } - - if (compiles_with_ecma_options(translated.transformed.value())) { - return true; - } - - // ECMA-262 places no width restriction on lookbehind assertions, while - // PCRE2 only accepts lookbehinds whose maximum width is bounded. As - // lookbehind and lookahead bodies follow the same grammar, the syntax - // check is retried with every lookbehind turned into a lookahead - const auto rewritten{ - lookbehinds_as_lookaheads(translated.transformed.value())}; - return rewritten != translated.transformed.value() && - compiles_with_ecma_options(rewritten); + return is_ecma262_pattern(pattern); } } // namespace sourcemeta::core diff --git a/test/regex/regex_is_ecma_test.cc b/test/regex/regex_is_ecma_test.cc index 887634ccc..8a74df299 100644 --- a/test/regex/regex_is_ecma_test.cc +++ b/test/regex/regex_is_ecma_test.cc @@ -1,6 +1,8 @@ #include #include +#include // std::string + TEST(suite_valid_basic) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("([abc])+\\s+$")); } @@ -93,6 +95,10 @@ TEST(valid_unicode_braced) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\u{1F600}")); } +TEST(valid_unicode_braced_out_of_range) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("\\u{110000}")); +} + TEST(valid_control_escape) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\cA")); } @@ -189,12 +195,12 @@ TEST(invalid_possessive_open_quantifier) { EXPECT_FALSE(sourcemeta::core::is_regex_ecma("a{3,}+")); } -TEST(valid_plus_after_literal_brace) { - EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a}+")); +TEST(invalid_plus_after_unescaped_brace) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("a}+")); } -TEST(valid_plus_after_escaped_brace_quantifier) { - EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a\\{2,3}+")); +TEST(invalid_plus_after_escaped_brace_quantifier) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("a\\{2,3}+")); } TEST(valid_variable_width_lookbehind) { @@ -257,8 +263,8 @@ TEST(invalid_inline_option_group) { EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?i)foo")); } -TEST(invalid_inline_option_scoped) { - EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?i:foo)")); +TEST(valid_inline_option_scoped) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?i:foo)")); } TEST(invalid_branch_reset_group) { @@ -289,8 +295,8 @@ TEST(invalid_quote_sequence) { EXPECT_FALSE(sourcemeta::core::is_regex_ecma("\\Qfoo\\E")); } -TEST(invalid_posix_class_alpha) { - EXPECT_FALSE(sourcemeta::core::is_regex_ecma("[[:alpha:]]")); +TEST(posix_class_alpha_reads_as_a_nested_class) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("[[:alpha:]]")); } TEST(invalid_possessive_quantifier) { @@ -411,6 +417,54 @@ TEST(escape_binary_prop) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{Alphabetic}")); } +TEST(escape_general_category_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{General_Category=Lu}")); +} + +TEST(escape_general_category_alias_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{gc=Lu}")); +} + +TEST(escape_general_category_long_value_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{gc=Uppercase_Letter}")); +} + +TEST(escape_negated_general_category_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\P{gc=Lu}")); +} + +TEST(escape_general_category_prop_in_class) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("[\\p{gc=Lu}]")); +} + +TEST(escape_general_category_prop_unknown_value) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("\\p{gc=NotAProperty}")); +} + +TEST(escape_general_category_prop_empty_value) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("\\p{gc=}")); +} + +TEST(escape_cased_letter_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{Cased_Letter}")); +} + +TEST(escape_combining_mark_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{Combining_Mark}")); +} + +TEST(escape_surrogate_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{Surrogate}")); +} + +TEST(escape_punct_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{punct}")); +} + +TEST(escape_cntrl_prop) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\p{cntrl}")); +} + TEST(escape_uxxxx) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("A")); } TEST(escape_escaped_slash) { @@ -430,15 +484,15 @@ TEST(inline_flag_flag_u) { } TEST(inline_flag_scoped_flag_on) { - EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?i:abc)")); + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?i:abc)")); } TEST(inline_flag_scoped_flag_off) { - EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?-i:abc)")); + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?-i:abc)")); } TEST(inline_flag_scoped_on_off) { - EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?i-s:abc)")); + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?i-s:abc)")); } TEST(inline_flag_verbose_extended) { @@ -551,6 +605,58 @@ TEST(quantifier_open_upper_control) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{2,}")); } +TEST(quantifier_exact_at_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{65535}")); +} + +TEST(quantifier_exact_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{65536}")); +} + +TEST(quantifier_exact_past_unsigned_range) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{4294967295}")); +} + +TEST(quantifier_exact_padded_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{0000070000}")); +} + +TEST(quantifier_interval_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{1,99999}")); +} + +TEST(quantifier_open_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a{99999,}")); +} + +TEST(quantifier_reversed_interval_past_pcre2_limit) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("a{99999,2}")); +} + +TEST(quantifier_reversed_interval_both_past_pcre2_limit) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("a{100000,99999}")); +} + +TEST(quantifier_class_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("[a-z]{70000}")); +} + +TEST(quantifier_group_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?:ab){70000}")); +} + +TEST(quantifier_capturing_group_past_pcre2_size) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(ab){10000}")); +} + +TEST(quantifier_escaped_braces_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("a\\{70000\\}")); +} + +TEST(quantifier_class_member_braces_past_pcre2_limit) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("[a{70000}]")); +} + TEST(quantifier_no_atom) { EXPECT_FALSE(sourcemeta::core::is_regex_ecma("{2}")); } @@ -602,3 +708,203 @@ TEST(redos_alternation_star_valid) { TEST(redos_repeated_group_valid) { EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(.*a){20}")); } + +TEST(modifier_group_add_flags) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?ims:abc)")); +} + +TEST(modifier_group_remove_flags) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?-ims:abc)")); +} + +TEST(modifier_group_both_empty) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?-:abc)")); +} + +TEST(modifier_group_repeated_flag) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?ii:abc)")); +} + +TEST(modifier_group_flag_on_both_sides) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?i-i:abc)")); +} + +TEST(modifier_group_unknown_flag) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?x:abc)")); +} + +TEST(group_name_leading_dollar_sign) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?<$foo>a)")); +} + +TEST(group_name_escaped_code_point) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?<\\u0041>a)")); +} + +TEST(group_name_outside_ascii) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?a)")); +} + +TEST(group_name_not_an_identifier) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?x)")); +} + +TEST(group_name_repeated_across_alternatives) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?a)|(?b)")); +} + +TEST(group_name_repeated_within_one_alternative) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?a)(?b)")); +} + +TEST(group_name_repeated_across_nested_alternatives) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?:(?a)|(?b))")); +} + +TEST(group_name_repeated_after_an_alternation) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?:(?a)|(?b))(?c)")); +} + +TEST(group_name_repeated_beside_an_alternation) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?:(?a)|(?b))|(?c)")); +} + +TEST(group_name_repeated_across_wrapped_alternatives) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("((?a))|(?b)")); +} + +TEST(group_name_repeated_within_its_own_group) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?(?a))")); +} + +TEST(group_name_repeated_within_its_own_alternation) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?a|(?b))")); +} + +TEST(group_name_repeated_through_an_escape) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?<\\u0041>a)(?b)")); +} + +TEST(group_name_repeated_through_an_escape_across_alternatives) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("(?<\\u0041>a)|(?b)")); +} + +TEST(named_backreference_without_a_group) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("\\k")); +} + +TEST(numbered_backreference_past_the_group_count) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(a)\\2")); +} + +TEST(lone_lead_surrogate_escape) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\uD83D")); +} + +TEST(surrogate_pair_escape) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("\\uD83D\\uDE00")); +} + +TEST(surrogate_pair_escape_in_class) { + EXPECT_TRUE(sourcemeta::core::is_regex_ecma("[\\uD83D\\uDE00]")); +} + +TEST(code_point_escape_past_the_unicode_range) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("\\u{110000}")); +} + +TEST(quantified_lookbehind) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?<=a)*")); +} + +TEST(quantified_negative_lookbehind) { + EXPECT_FALSE(sourcemeta::core::is_regex_ecma("(?