Source code
Revision control
Copy as Markdown
Other Tools
# These test the UTF-8 modes expose by regex-automata. Namely, when utf8 is
# true, then we promise that the haystack is valid UTF-8. (Otherwise behavior
# is unspecified.) This also corresponds to building the regex engine with the
# following two guarantees:
#
# 1) For any non-empty match reported, its span is guaranteed to correspond to
# valid UTF-8.
# 2) All empty or zero-width matches reported must never split a UTF-8
# encoded codepoint. If the haystack has invalid UTF-8, then this results in
# unspecified behavior.
#
# The (2) is in particular what we focus our testing on since (1) is generally
# guaranteed by regex-syntax's AST-to-HIR translator and is well tested there.
# The thing with (2) is that it can't be described in the HIR, so the regex
# engines have to handle that case. Thus, we test it here.
#
# Note that it is possible to build a regex that has property (1) but not
# (2), and vice versa. This is done by building the HIR with 'utf8=true' but
# building the Thompson NFA with 'utf8=false'. We don't test that here because
# the harness doesn't expose a way to enable or disable UTF-8 mode with that
# granularity. Instead, those combinations are lightly tested via doc examples.
# That's not to say that (1) without (2) is uncommon. Indeed, ripgrep uses it
# because it cannot guarantee that its haystack is valid UTF-8.
# This tests that an empty regex doesn't split a codepoint.
[[test]]
name = "empty-utf8yes"
regex = ''
haystack = '☃'
matches = [[0, 0], [3, 3]]
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-overlapping"
regex = ''
haystack = '☃'
matches = [[0, 0], [3, 3]]
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# This tests that an empty regex DOES split a codepoint when utf=false.
[[test]]
name = "empty-utf8no"
regex = ''
haystack = '☃'
matches = [[0, 0], [1, 1], [2, 2], [3, 3]]
unicode = true
utf8 = false
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8no-overlapping"
regex = ''
haystack = '☃'
matches = [[0, 0], [1, 1], [2, 2], [3, 3]]
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"
# This tests that an empty regex doesn't split a codepoint, even if we give
# it bounds entirely within the codepoint.
#
# This is one of the trickier cases and is what motivated the current UTF-8
# mode design. In particular, at one point, this test failed the 'is_match'
# variant of the test but not 'find'. This is because the 'is_match' code path
# is specifically optimized for "was a match found" rather than "where is the
# match." In the former case, you don't really care about the empty-vs-non-empty
# matches, and thus, the codepoint splitting filtering logic wasn't getting
# applied. (In multiple ways across multiple regex engines.) In this way, you
# can wind up with a situation where 'is_match' says "yes," but 'find' says,
# "I didn't find anything." Which is... not great.
#
# I could have decided to say that providing boundaries that themselves split
# a codepoint would have unspecified behavior. But I couldn't quite convince
# myself that such boundaries were the only way to get an inconsistency between
# 'is_match' and 'find'.
#
# Note that I also tried to come up with a test like this that fails without
# using `bounds`. Specifically, a test where 'is_match' and 'find' disagree.
# But I couldn't do it, and I'm tempted to conclude it is impossible. The
# fundamental problem is that you need to simultaneously produce an empty match
# that splits a codepoint while *not* matching before or after the codepoint.
[[test]]
name = "empty-utf8yes-bounds"
regex = ''
haystack = '𝛃'
bounds = [1, 3]
matches = []
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-bounds-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 3]
matches = []
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# This tests that an empty regex splits a codepoint when the bounds are
# entirely within the codepoint.
[[test]]
name = "empty-utf8no-bounds"
regex = ''
haystack = '𝛃'
bounds = [1, 3]
matches = [[1, 1], [2, 2], [3, 3]]
unicode = true
utf8 = false
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8no-bounds-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 3]
matches = [[1, 1], [2, 2], [3, 3]]
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"
# In this test, we anchor the search. Since the start position is also a UTF-8
# boundary, we get a match.
[[test]]
name = "empty-utf8yes-anchored"
regex = ''
haystack = '𝛃'
matches = [[0, 0]]
anchored = true
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-anchored-overlapping"
regex = ''
haystack = '𝛃'
matches = [[0, 0]]
anchored = true
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# Same as above, except with UTF-8 mode disabled. It almost doesn't change the
# result, except for the fact that since this is an anchored search and we
# always find all matches, the test harness will keep reporting matches until
# none are found. Because it's anchored, matches will be reported so long as
# they are directly adjacent. Since with UTF-8 mode the next anchored search
# after the match at [0, 0] fails, iteration stops (and doesn't find the last
# match at [4, 4]).
[[test]]
name = "empty-utf8no-anchored"
regex = ''
haystack = '𝛃'
matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]]
anchored = true
unicode = true
utf8 = false
# Tests the overlapping case of the above.
#
# Note that overlapping anchored searches are a little weird, and it's not
# totally clear what their semantics ought to be. For now, we just test the
# current behavior of our test shim that implements overlapping search. (This
# is one of the reasons why we don't really expose regex-level overlapping
# searches.)
[[test]]
name = "empty-utf8no-anchored-overlapping"
regex = ''
haystack = '𝛃'
matches = [[0, 0]]
anchored = true
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"
# In this test, we anchor the search, but also set bounds. The bounds start the
# search in the middle of a codepoint, so there should never be a match.
[[test]]
name = "empty-utf8yes-anchored-bounds"
regex = ''
haystack = '𝛃'
matches = []
bounds = [1, 3]
anchored = true
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-anchored-bounds-overlapping"
regex = ''
haystack = '𝛃'
matches = []
bounds = [1, 3]
anchored = true
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# Same as above, except with UTF-8 mode disabled. Without UTF-8 mode enabled,
# matching within a codepoint is allowed. And remember, as in the anchored test
# above with UTF-8 mode disabled, iteration will report all adjacent matches.
# The matches at [0, 0] and [4, 4] are not included because of the bounds of
# the search.
[[test]]
name = "empty-utf8no-anchored-bounds"
regex = ''
haystack = '𝛃'
bounds = [1, 3]
matches = [[1, 1], [2, 2], [3, 3]]
anchored = true
unicode = true
utf8 = false
# Tests the overlapping case of the above.
#
# Note that overlapping anchored searches are a little weird, and it's not
# totally clear what their semantics ought to be. For now, we just test the
# current behavior of our test shim that implements overlapping search. (This
# is one of the reasons why we don't really expose regex-level overlapping
# searches.)
[[test]]
name = "empty-utf8no-anchored-bounds-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 3]
matches = [[1, 1]]
anchored = true
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"
# This tests that we find the match at the end of the string when the bounds
# exclude the first match.
[[test]]
name = "empty-utf8yes-startbound"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = [[4, 4]]
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-startbound-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = [[4, 4]]
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# Same as above, except since UTF-8 mode is disabled, we also find the matches
# inbetween that split the codepoint.
[[test]]
name = "empty-utf8no-startbound"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = [[1, 1], [2, 2], [3, 3], [4, 4]]
unicode = true
utf8 = false
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8no-startbound-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = [[1, 1], [2, 2], [3, 3], [4, 4]]
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"
# This tests that we don't find any matches in an anchored search, even when
# the bounds include a match (at the end).
[[test]]
name = "empty-utf8yes-anchored-startbound"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = []
anchored = true
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-anchored-startbound-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = []
anchored = true
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# Same as above, except since UTF-8 mode is disabled, we also find the matches
# inbetween that split the codepoint. Even though this is an anchored search,
# since the matches are adjacent, we find all of them.
[[test]]
name = "empty-utf8no-anchored-startbound"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = [[1, 1], [2, 2], [3, 3], [4, 4]]
anchored = true
unicode = true
utf8 = false
# Tests the overlapping case of the above.
#
# Note that overlapping anchored searches are a little weird, and it's not
# totally clear what their semantics ought to be. For now, we just test the
# current behavior of our test shim that implements overlapping search. (This
# is one of the reasons why we don't really expose regex-level overlapping
# searches.)
[[test]]
name = "empty-utf8no-anchored-startbound-overlapping"
regex = ''
haystack = '𝛃'
bounds = [1, 4]
matches = [[1, 1]]
anchored = true
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"
# This tests that we find the match at the end of the haystack in UTF-8 mode
# when our bounds only include the empty string at the end of the haystack.
[[test]]
name = "empty-utf8yes-anchored-endbound"
regex = ''
haystack = '𝛃'
bounds = [4, 4]
matches = [[4, 4]]
anchored = true
unicode = true
utf8 = true
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8yes-anchored-endbound-overlapping"
regex = ''
haystack = '𝛃'
bounds = [4, 4]
matches = [[4, 4]]
anchored = true
unicode = true
utf8 = true
match-kind = "all"
search-kind = "overlapping"
# Same as above, but with UTF-8 mode disabled. Results remain the same since
# the only possible match does not split a codepoint.
[[test]]
name = "empty-utf8no-anchored-endbound"
regex = ''
haystack = '𝛃'
bounds = [4, 4]
matches = [[4, 4]]
anchored = true
unicode = true
utf8 = false
# Tests the overlapping case of the above.
[[test]]
name = "empty-utf8no-anchored-endbound-overlapping"
regex = ''
haystack = '𝛃'
bounds = [4, 4]
matches = [[4, 4]]
anchored = true
unicode = true
utf8 = false
match-kind = "all"
search-kind = "overlapping"