5#ifndef ADA_URL_PATTERN_HELPERS_INL_H
6#define ADA_URL_PATTERN_HELPERS_INL_H
17#include "ada/expected.h"
21#if ADA_INCLUDE_URL_PATTERN
22namespace ada::url_pattern_helpers {
23#if defined(ADA_TESTING) || defined(ADA_LOGGING)
24inline std::string
to_string(token_type type) {
26 case token_type::INVALID_CHAR:
27 return "INVALID_CHAR";
28 case token_type::OPEN:
30 case token_type::CLOSE:
32 case token_type::REGEXP:
34 case token_type::NAME:
36 case token_type::CHAR:
38 case token_type::ESCAPED_CHAR:
39 return "ESCAPED_CHAR";
40 case token_type::OTHER_MODIFIER:
41 return "OTHER_MODIFIER";
42 case token_type::ASTERISK:
52template <url_pattern_regex::regex_concept regex_prov
ider>
53constexpr void constructor_string_parser<regex_provider>::rewind() {
55 token_index = component_start;
60template <url_pattern_regex::regex_concept regex_prov
ider>
61constexpr bool constructor_string_parser<regex_provider>::is_hash_prefix() {
64 return is_non_special_pattern_char(token_index,
'#');
67template <url_pattern_regex::regex_concept regex_prov
ider>
68constexpr bool constructor_string_parser<regex_provider>::is_search_prefix() {
71 if (is_non_special_pattern_char(token_index,
'?')) {
77 if (token_list[token_index].value !=
"?") {
82 if (token_index == 0)
return true;
84 auto previous_index = token_index - 1;
87 auto previous_token = get_safe_token(previous_index);
94 return !(previous_token->type == token_type::NAME ||
95 previous_token->type == token_type::REGEXP ||
96 previous_token->type == token_type::CLOSE ||
97 previous_token->type == token_type::ASTERISK);
100template <url_pattern_regex::regex_concept regex_prov
ider>
102constructor_string_parser<regex_provider>::is_non_special_pattern_char(
103 size_t index, uint32_t value)
const {
105 auto token = get_safe_token(index);
111 if (!token->value.empty() &&
112 static_cast<uint32_t
>(token->value[0]) != value) {
121 return token->type == token_type::CHAR ||
122 token->type == token_type::ESCAPED_CHAR ||
123 token->type == token_type::INVALID_CHAR;
126template <url_pattern_regex::regex_concept regex_prov
ider>
127constexpr const token*
128constructor_string_parser<regex_provider>::get_safe_token(
size_t index)
const {
131 if (index < token_list.size()) [[likely]] {
132 return &token_list[index];
143 return &token_list.back();
146template <url_pattern_regex::regex_concept regex_prov
ider>
147constexpr bool constructor_string_parser<regex_provider>::is_group_open()
151 return token_list[token_index].type == token_type::OPEN;
154template <url_pattern_regex::regex_concept regex_prov
ider>
155constexpr bool constructor_string_parser<regex_provider>::is_group_close()
159 return token_list[token_index].type == token_type::CLOSE;
162template <url_pattern_regex::regex_concept regex_prov
ider>
164constructor_string_parser<regex_provider>::next_is_authority_slashes()
const {
167 if (!is_non_special_pattern_char(token_index + 1,
'/')) {
172 if (!is_non_special_pattern_char(token_index + 2,
'/')) {
178template <url_pattern_regex::regex_concept regex_prov
ider>
179constexpr bool constructor_string_parser<regex_provider>::is_protocol_suffix()
183 return is_non_special_pattern_char(token_index,
':');
186template <url_pattern_regex::regex_concept regex_prov
ider>
187void constructor_string_parser<regex_provider>::change_state(State new_state,
192 if (state != State::INIT && state != State::AUTHORITY &&
193 state != State::DONE) {
194 auto value = make_component_string();
197 case State::PROTOCOL: {
201 case State::USERNAME: {
205 case State::PASSWORD: {
209 case State::HOSTNAME: {
217 case State::PATHNAME: {
221 case State::SEARCH: {
235 if (state != State::INIT && new_state != State::DONE) {
240 if ((state == State::PROTOCOL || state == State::AUTHORITY ||
241 state == State::USERNAME || state == State::PASSWORD) &&
242 (new_state == State::PORT || new_state == State::PATHNAME ||
243 new_state == State::SEARCH || new_state == State::HASH) &&
251 if ((state == State::PROTOCOL || state == State::AUTHORITY ||
252 state == State::USERNAME || state == State::PASSWORD ||
253 state == State::HOSTNAME || state == State::PORT) &&
254 (new_state == State::SEARCH || new_state == State::HASH) &&
256 if (protocol_matches_a_special_scheme_flag) {
268 if ((state == State::PROTOCOL || state == State::AUTHORITY ||
269 state == State::USERNAME || state == State::PASSWORD ||
270 state == State::HOSTNAME || state == State::PORT ||
271 state == State::PATHNAME) &&
272 new_state == State::HASH && !
result.search) {
281 component_start = token_index;
286template <url_pattern_regex::regex_concept regex_prov
ider>
287std::string constructor_string_parser<regex_provider>::make_component_string() {
293 const auto end_index = token_list[token_index].index;
296 const auto component_start_token = get_safe_token(component_start);
299 const auto component_start_input_index = component_start_token->index;
302 return std::string(input.substr(component_start_input_index,
303 end_index - component_start_input_index));
306template <url_pattern_regex::regex_concept regex_prov
ider>
308constructor_string_parser<regex_provider>::is_an_identity_terminator()
const {
311 return is_non_special_pattern_char(token_index,
'@');
314template <url_pattern_regex::regex_concept regex_prov
ider>
315constexpr bool constructor_string_parser<regex_provider>::is_pathname_start()
319 return is_non_special_pattern_char(token_index,
'/');
322template <url_pattern_regex::regex_concept regex_prov
ider>
323constexpr bool constructor_string_parser<regex_provider>::is_password_prefix()
327 return is_non_special_pattern_char(token_index,
':');
330template <url_pattern_regex::regex_concept regex_prov
ider>
331constexpr bool constructor_string_parser<regex_provider>::is_an_ipv6_open()
335 return is_non_special_pattern_char(token_index,
'[');
338template <url_pattern_regex::regex_concept regex_prov
ider>
339constexpr bool constructor_string_parser<regex_provider>::is_an_ipv6_close()
343 return is_non_special_pattern_char(token_index,
']');
346template <url_pattern_regex::regex_concept regex_prov
ider>
347constexpr bool constructor_string_parser<regex_provider>::is_port_prefix()
351 return is_non_special_pattern_char(token_index,
':');
354constexpr void Tokenizer::get_next_code_point() {
355 ada_log(
"Tokenizer::get_next_code_point called with index=", next_index);
360 invalid_code_point =
false;
362 size_t number_bytes = 0;
363 const size_t initial_index = next_index;
364 unsigned char first_byte = input[next_index];
366 if ((first_byte & 0x80) == 0) {
369 code_point = first_byte;
370 ada_log(
"Tokenizer::get_next_code_point returning ASCII code point=",
371 uint32_t(code_point));
372 ada_log(
"Tokenizer::get_next_code_point next_index =", next_index,
373 " input.size()=", input.size());
376 ada_log(
"Tokenizer::get_next_code_point read first byte=",
377 uint32_t(first_byte));
378 if ((first_byte & 0xE0) == 0xC0) {
379 code_point = first_byte & 0x1F;
381 ada_log(
"Tokenizer::get_next_code_point two bytes");
382 }
else if ((first_byte & 0xF0) == 0xE0) {
383 code_point = first_byte & 0x0F;
385 ada_log(
"Tokenizer::get_next_code_point three bytes");
386 }
else if ((first_byte & 0xF8) == 0xF0) {
387 code_point = first_byte & 0x07;
389 ada_log(
"Tokenizer::get_next_code_point four bytes");
393 if ((number_bytes == 2 && first_byte < 0xC2) ||
394 (number_bytes == 4 && first_byte > 0xF4)) {
395 invalid_code_point =
true;
396 code_point = first_byte;
397 next_index = initial_index + 1;
402 if (number_bytes == 0) {
403 invalid_code_point =
true;
404 code_point = first_byte;
405 next_index = initial_index + 1;
410 if (number_bytes + next_index > input.size()) {
411 invalid_code_point =
true;
412 code_point = first_byte;
413 next_index = initial_index + 1;
417 for (
size_t i = 1 + next_index; i < number_bytes + next_index; ++i) {
418 unsigned char byte = input[i];
419 if ((
byte & 0xC0) != 0x80) {
420 invalid_code_point =
true;
421 code_point = first_byte;
422 next_index = initial_index + 1;
425 ada_log(
"Tokenizer::get_next_code_point read byte=", uint32_t(
byte));
426 code_point = (code_point << 6) | (
byte & 0x3F);
428 ada_log(
"Tokenizer::get_next_code_point returning non-ASCII code point=",
429 uint32_t(code_point));
430 ada_log(
"Tokenizer::get_next_code_point next_index =", next_index,
431 " input.size()=", input.size());
432 next_index += number_bytes;
435constexpr void Tokenizer::seek_and_get_next_code_point(
size_t new_index) {
436 ada_log(
"Tokenizer::seek_and_get_next_code_point called with new_index=",
439 next_index = new_index;
441 get_next_code_point();
444inline void Tokenizer::add_token(token_type type,
size_t next_position,
445 size_t value_position,
size_t value_length) {
446 ada_log(
"Tokenizer::add_token called with type=",
to_string(type),
447 " next_position=", next_position,
" value_position=", value_position);
456 token_list.emplace_back(type, index,
457 input.substr(value_position, value_length));
459 index = next_position;
462inline void Tokenizer::add_token_with_default_length(token_type type,
463 size_t next_position,
464 size_t value_position) {
466 auto computed_length = next_position - value_position;
469 add_token(type, next_position, value_position, computed_length);
472inline void Tokenizer::add_token_with_defaults(token_type type) {
473 ada_log(
"Tokenizer::add_token_with_defaults called with type=",
477 add_token_with_default_length(type, next_index, index);
481Tokenizer::process_tokenizing_error(
size_t next_position,
482 size_t value_position) {
484 if (policy == token_policy::strict) {
485 ada_log(
"process_tokenizing_error failed with next_position=",
486 next_position,
" value_position=", value_position);
487 return errors::type_error;
493 add_token_with_default_length(token_type::INVALID_CHAR, next_position,
498template <url_pattern_encoding_callback F>
499token* url_pattern_parser<F>::try_consume_modifier_token() {
502 auto token = try_consume_token(token_type::OTHER_MODIFIER);
504 if (token)
return token;
508 return try_consume_token(token_type::ASTERISK);
511template <url_pattern_encoding_callback F>
512token* url_pattern_parser<F>::try_consume_regexp_or_wildcard_token(
513 const token* name_token) {
516 auto token = try_consume_token(token_type::REGEXP);
519 if (!name_token && !token) {
520 token = try_consume_token(token_type::ASTERISK);
526template <url_pattern_encoding_callback F>
527token* url_pattern_parser<F>::try_consume_token(token_type type) {
528 ada_log(
"url_pattern_parser::try_consume_token called with type=",
533 auto& next_token = tokens[index];
535 if (next_token.type != type)
return nullptr;
542template <url_pattern_encoding_callback F>
543std::string url_pattern_parser<F>::consume_text() {
550 auto token = try_consume_token(token_type::CHAR);
553 if (!token) token = try_consume_token(token_type::ESCAPED_CHAR);
557 result.append(token->value);
563template <url_pattern_encoding_callback F>
564bool url_pattern_parser<F>::consume_required_token(token_type type) {
565 ada_log(
"url_pattern_parser::consume_required_token called with type=",
569 return try_consume_token(type) !=
nullptr;
572template <url_pattern_encoding_callback F>
574url_pattern_parser<F>::maybe_add_part_from_the_pending_fixed_value() {
576 if (pending_fixed_value.empty()) {
577 ada_log(
"pending_fixed_value is empty");
582 auto encoded_value = encoding_callback(pending_fixed_value);
583 if (!encoded_value) {
584 ada_log(
"failed to encode pending_fixed_value: ", pending_fixed_value);
585 return encoded_value.error();
588 pending_fixed_value.clear();
592 parts.emplace_back(url_pattern_part_type::FIXED_TEXT,
593 std::move(*encoded_value),
594 url_pattern_part_modifier::none);
598template <url_pattern_encoding_callback F>
599std::optional<errors> url_pattern_parser<F>::add_part(
600 std::string_view prefix, token* name_token, token* regexp_or_wildcard_token,
601 std::string_view suffix, token* modifier_token) {
603 auto modifier = url_pattern_part_modifier::none;
605 if (modifier_token) {
607 if (modifier_token->value ==
"?") {
608 modifier = url_pattern_part_modifier::optional;
609 }
else if (modifier_token->value ==
"*") {
612 modifier = url_pattern_part_modifier::zero_or_more;
613 }
else if (modifier_token->value ==
"+") {
616 modifier = url_pattern_part_modifier::one_or_more;
621 if (!name_token && !regexp_or_wildcard_token &&
622 modifier == url_pattern_part_modifier::none) {
624 pending_fixed_value.append(prefix);
628 if (
auto error = maybe_add_part_from_the_pending_fixed_value()) {
632 if (!name_token && !regexp_or_wildcard_token) {
636 if (prefix.empty())
return std::nullopt;
639 auto encoded_value = encoding_callback(prefix);
640 if (!encoded_value) {
641 return encoded_value.error();
646 parts.emplace_back(url_pattern_part_type::FIXED_TEXT,
647 std::move(*encoded_value), modifier);
651 std::string regexp_value{};
654 if (!regexp_or_wildcard_token) {
655 regexp_value = segment_wildcard_regexp;
656 }
else if (regexp_or_wildcard_token->type == token_type::ASTERISK) {
662 regexp_value = regexp_or_wildcard_token->value;
665 auto type = url_pattern_part_type::REGEXP;
667 if (regexp_value == segment_wildcard_regexp) {
669 type = url_pattern_part_type::SEGMENT_WILDCARD;
671 regexp_value.clear();
672 }
else if (regexp_value ==
".*") {
675 type = url_pattern_part_type::FULL_WILDCARD;
677 regexp_value.clear();
683 name = name_token->value;
684 }
else if (regexp_or_wildcard_token !=
nullptr) {
687 name = std::to_string(next_numeric_name);
695 if (!name.empty() && !seen_names.insert(name).second) {
696 return errors::type_error;
700 auto encoded_prefix = encoding_callback(prefix);
701 if (!encoded_prefix)
return encoded_prefix.error();
704 auto encoded_suffix = encoding_callback(suffix);
705 if (!encoded_suffix)
return encoded_suffix.error();
710 parts.emplace_back(type, std::move(regexp_value), modifier, std::move(name),
711 std::move(*encoded_prefix), std::move(*encoded_suffix));
715template <url_pattern_encoding_callback F>
716tl::expected<std::vector<url_pattern_part>,
errors> parse_pattern_string(
717 std::string_view input, url_pattern_compile_component_options& options,
718 F& encoding_callback) {
719 ada_log(
"parse_pattern_string input=", input);
723 auto parser = url_pattern_parser<F>(
724 encoding_callback, generate_segment_wildcard_regexp(options));
727 auto tokenize_result = tokenize(input, token_policy::strict);
728 if (!tokenize_result) {
729 ada_log(
"parse_pattern_string tokenize failed");
730 return tl::unexpected(tokenize_result.error());
732 parser.tokens = std::move(*tokenize_result);
735 while (parser.can_continue()) {
738 auto char_token = parser.try_consume_token(token_type::CHAR);
741 auto name_token = parser.try_consume_token(token_type::NAME);
744 auto regexp_or_wildcard_token =
745 parser.try_consume_regexp_or_wildcard_token(name_token);
747 if (name_token || regexp_or_wildcard_token) {
749 std::string prefix{};
751 if (char_token) prefix = char_token->value;
753 if (!prefix.empty() && prefix != options.get_prefix()) {
755 parser.pending_fixed_value.append(prefix);
760 if (
auto error = parser.maybe_add_part_from_the_pending_fixed_value()) {
761 ada_log(
"maybe_add_part_from_the_pending_fixed_value failed");
762 return tl::unexpected(*error);
766 auto modifier_token = parser.try_consume_modifier_token();
770 parser.add_part(prefix, name_token, regexp_or_wildcard_token,
"",
772 ada_log(
"parser.add_part failed");
773 return tl::unexpected(*error);
780 auto fixed_token = char_token;
784 fixed_token = parser.try_consume_token(token_type::ESCAPED_CHAR);
788 parser.pending_fixed_value.append(fixed_token->value);
794 auto open_token = parser.try_consume_token(token_type::OPEN);
798 auto prefix_ = parser.consume_text();
801 name_token = parser.try_consume_token(token_type::NAME);
804 regexp_or_wildcard_token =
805 parser.try_consume_regexp_or_wildcard_token(name_token);
807 auto suffix_ = parser.consume_text();
809 if (!parser.consume_required_token(token_type::CLOSE)) {
810 ada_log(
"parser.consume_required_token failed");
811 return tl::unexpected(errors::type_error);
815 auto modifier_token = parser.try_consume_modifier_token();
819 parser.add_part(prefix_, name_token, regexp_or_wildcard_token,
820 suffix_, modifier_token)) {
821 return tl::unexpected(*error);
827 if (
auto error = parser.maybe_add_part_from_the_pending_fixed_value()) {
828 ada_log(
"maybe_add_part_from_the_pending_fixed_value failed on line 992");
829 return tl::unexpected(*error);
832 if (!parser.consume_required_token(token_type::END)) {
833 return tl::unexpected(errors::type_error);
836 ada_log(
"parser.parts size is: ", parser.parts.size());
841template <url_pattern_regex::regex_concept regex_prov
ider>
842bool protocol_component_matches_special_scheme(
843 url_pattern_component<regex_provider>& component) {
845 switch (component.type) {
846 case url_pattern_component_type::EMPTY:
849 case url_pattern_component_type::EXACT_MATCH:
851 return component.exact_match_value ==
"http" ||
852 component.exact_match_value ==
"https" ||
853 component.exact_match_value ==
"ws" ||
854 component.exact_match_value ==
"wss" ||
855 component.exact_match_value ==
"ftp" ||
856 component.exact_match_value ==
"file";
857 case url_pattern_component_type::FULL_WILDCARD:
860 case url_pattern_component_type::REGEXP:
862 auto& regex = component.regexp;
863 return regex_provider::regex_match(
"http", regex) ||
864 regex_provider::regex_match(
"https", regex) ||
865 regex_provider::regex_match(
"ws", regex) ||
866 regex_provider::regex_match(
"wss", regex) ||
867 regex_provider::regex_match(
"ftp", regex) ||
868 regex_provider::regex_match(
"file", regex);
873template <url_pattern_regex::regex_concept regex_prov
ider>
874inline std::optional<errors> constructor_string_parser<
877 "constructor_string_parser::compute_protocol_matches_special_scheme_"
881 auto protocol_string = make_component_string();
884 auto protocol_component = url_pattern_component<regex_provider>::compile(
885 protocol_string, canonicalize_protocol,
886 url_pattern_compile_component_options::DEFAULT);
887 if (!protocol_component) {
888 ada_log(
"url_pattern_component::compile failed for protocol_string ",
890 return protocol_component.error();
895 if (protocol_component_matches_special_scheme(*protocol_component)) {
896 protocol_matches_a_special_scheme_flag =
true;
901template <url_pattern_regex::regex_concept regex_prov
ider>
902tl::expected<url_pattern_init, errors>
903constructor_string_parser<regex_provider>::parse(std::string_view input) {
904 ada_log(
"constructor_string_parser::parse input=", input);
907 auto token_list = tokenize(input, token_policy::lenient);
909 return tl::unexpected(token_list.error());
911 auto parser = constructor_string_parser(input, std::move(*token_list));
914 while (parser.token_index < parser.token_list.size()) {
916 parser.token_increment = 1;
919 if (parser.token_list[parser.token_index].type == token_type::END) {
921 if (parser.state == State::INIT) {
926 if (parser.is_hash_prefix()) {
927 parser.change_state(State::HASH, 1);
928 }
else if (parser.is_search_prefix()) {
931 parser.change_state(State::SEARCH, 1);
934 parser.change_state(State::PATHNAME, 0);
937 parser.token_index += parser.token_increment;
942 if (parser.state == State::AUTHORITY) {
946 parser.change_state(State::HOSTNAME, 0);
948 parser.token_index += parser.token_increment;
954 parser.change_state(State::DONE, 0);
960 if (parser.is_group_open()) {
962 parser.group_depth += 1;
964 parser.token_index += parser.token_increment;
970 if (parser.group_depth > 0) {
973 if (parser.is_group_close()) {
974 parser.group_depth -= 1;
977 parser.token_index += parser.token_increment;
983 switch (parser.state) {
986 if (parser.is_protocol_suffix()) {
989 parser.change_state(State::PROTOCOL, 0);
993 case State::PROTOCOL: {
995 if (parser.is_protocol_suffix()) {
997 if (
const auto error =
998 parser.compute_protocol_matches_special_scheme_flag()) {
999 ada_log(
"compute_protocol_matches_special_scheme_flag failed");
1000 return tl::unexpected(*error);
1003 auto next_state = State::PATHNAME;
1008 if (parser.next_is_authority_slashes()) {
1010 next_state = State::AUTHORITY;
1013 }
else if (parser.protocol_matches_a_special_scheme_flag) {
1016 next_state = State::AUTHORITY;
1020 parser.change_state(next_state, skip);
1024 case State::AUTHORITY: {
1027 if (parser.is_an_identity_terminator()) {
1029 parser.change_state(State::USERNAME, 0);
1030 }
else if (parser.is_pathname_start() || parser.is_search_prefix() ||
1031 parser.is_hash_prefix()) {
1038 parser.change_state(State::HOSTNAME, 0);
1042 case State::USERNAME: {
1045 if (parser.is_password_prefix()) {
1046 parser.change_state(State::PASSWORD, 1);
1047 }
else if (parser.is_an_identity_terminator()) {
1051 parser.change_state(State::HOSTNAME, 1);
1055 case State::PASSWORD: {
1058 if (parser.is_an_identity_terminator()) {
1059 parser.change_state(State::HOSTNAME, 1);
1063 case State::HOSTNAME: {
1066 if (parser.is_an_ipv6_open()) {
1067 parser.hostname_ipv6_bracket_depth += 1;
1068 }
else if (parser.is_an_ipv6_close()) {
1071 parser.hostname_ipv6_bracket_depth -= 1;
1072 }
else if (parser.is_port_prefix() &&
1073 parser.hostname_ipv6_bracket_depth == 0) {
1077 parser.change_state(State::PORT, 1);
1078 }
else if (parser.is_pathname_start()) {
1081 parser.change_state(State::PATHNAME, 0);
1082 }
else if (parser.is_search_prefix()) {
1085 parser.change_state(State::SEARCH, 1);
1086 }
else if (parser.is_hash_prefix()) {
1089 parser.change_state(State::HASH, 1);
1097 if (parser.is_pathname_start()) {
1098 parser.change_state(State::PATHNAME, 0);
1099 }
else if (parser.is_search_prefix()) {
1102 parser.change_state(State::SEARCH, 1);
1103 }
else if (parser.is_hash_prefix()) {
1106 parser.change_state(State::HASH, 1);
1110 case State::PATHNAME: {
1113 if (parser.is_search_prefix()) {
1114 parser.change_state(State::SEARCH, 1);
1115 }
else if (parser.is_hash_prefix()) {
1118 parser.change_state(State::HASH, 1);
1122 case State::SEARCH: {
1125 if (parser.is_hash_prefix()) {
1126 parser.change_state(State::HASH, 1);
1141 parser.token_index += parser.token_increment;
1146 if (parser.result.hostname && !parser.result.port) {
1147 parser.result.port =
"";
1151 return parser.result;
Cross-platform compiler macros and common definitions.
#define ADA_ASSERT_TRUE(COND)
User-facing functions for URL parsing and manipulation.
type
Enumeration of URL scheme types.
errors
Error codes for URL parsing operations.
state
States in the URL parsing state machine.
ada_warn_unused std::string_view to_string(encoding_type type)
tl::expected< result_type, ada::errors > result
ada::url_pattern_regex::std_regex_provider regex_provider
Declaration for the URLPattern helpers.