[PERF] Remove \b anchor from escaped-unicode codePointPat
The leading \b blocked the regexp engine's literal-prefix fast path, so codePointPat was tried at every offset in every chunk. Apply the word-boundary check in Go instead, gate each pattern on its fixed literal anchor, and let FromChunk reuse the match list it already computed rather than scanning a second time to decode. Adds an equivalence test comparing the new implementation against the old one on generated, hand-picked, and fuzzed inputs.
This commit is contained in:
Ahsan-Sarbaz
@@ -19,7 +19,14 @@ var _ Decoder = (*EscapedUnicode)(nil)
|
||||
var (
|
||||
// Standard Unicode notation.
|
||||
//https://unicode.org/standard/principles.html
|
||||
codePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)
|
||||
//
|
||||
// The word boundary that belongs at the front of this pattern is applied by
|
||||
// codePointMatches instead of being written here. A leading empty-width
|
||||
// assertion stops regexp from extracting a literal prefix, and without a
|
||||
// prefix every offset in the chunk has to be tried rather than skipping
|
||||
// straight to the next "U+". That one difference made this pattern cost
|
||||
// more than every other pattern in this file combined.
|
||||
codePointPat = regexp.MustCompile(`U\+([a-fA-F0-9]{4}).?`)
|
||||
|
||||
// Common escape sequence used in programming languages.
|
||||
escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`)
|
||||
@@ -52,6 +59,53 @@ var (
|
||||
// hexEscapePat = regexp.MustCompile(`0x([a-fA-F0-9]{1,6})(?:\s|$)`)
|
||||
)
|
||||
|
||||
// Every escape form this decoder understands starts with a fixed literal: a
|
||||
// backslash for the \uXXXX family and CSS escapes, "U+" for standard code
|
||||
// point notation, "&#x" for HTML and "%u" for percent encoding. Looking for
|
||||
// that literal is far cheaper than running the pattern, and a chunk without it
|
||||
// cannot match, so each pattern is only run once its anchor is present.
|
||||
var (
|
||||
codePointAnchor = []byte("U+")
|
||||
htmlAnchor = []byte("&#x")
|
||||
percentAnchor = []byte("%u")
|
||||
)
|
||||
|
||||
// isWordByte reports whether c is a word character as \b defines it: ASCII
|
||||
// letters, digits and underscore.
|
||||
func isWordByte(c byte) bool {
|
||||
return c == '_' ||
|
||||
('0' <= c && c <= '9') ||
|
||||
('a' <= c && c <= 'z') ||
|
||||
('A' <= c && c <= 'Z')
|
||||
}
|
||||
|
||||
// codePointMatches returns the submatch indices of every U+XXXX sequence in
|
||||
// input that starts on a word boundary.
|
||||
//
|
||||
// This is the check the leading \b used to perform. "U" is itself a word
|
||||
// character, so the boundary holds exactly when the sequence starts the input
|
||||
// or follows a byte that is not a word character.
|
||||
//
|
||||
// Dropping a match here can never hide another one. A second "U+" cannot begin
|
||||
// inside the four hex digits, since "+" is not a hex digit, so the only overlap
|
||||
// possible is the trailing optional byte covering a following "U". That "U"
|
||||
// is preceded by a hex digit, which is a word character, so the boundary would
|
||||
// have rejected it as well.
|
||||
func codePointMatches(input []byte) [][]int {
|
||||
if !bytes.Contains(input, codePointAnchor) {
|
||||
return nil
|
||||
}
|
||||
all := codePointPat.FindAllSubmatchIndex(input, -1)
|
||||
kept := all[:0]
|
||||
for _, m := range all {
|
||||
if m[0] > 0 && isWordByte(input[m[0]-1]) {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, m)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
func (d *EscapedUnicode) Type() detectorspb.DecoderType {
|
||||
return detectorspb.DecoderType_ESCAPED_UNICODE
|
||||
}
|
||||
@@ -70,29 +124,31 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
// Process patterns in priority order - more specific patterns first
|
||||
// This prevents conflicts where multiple patterns match the same input
|
||||
|
||||
hasBackslash := bytes.IndexByte(chunkData, '\\') >= 0
|
||||
|
||||
// Long escape format (8 hex digits) - highest priority
|
||||
if longEscapePat.Match(chunkData) {
|
||||
if hasBackslash && longEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeLongEscape(chunkData)
|
||||
} else if braceEscapePat.Match(chunkData) {
|
||||
} else if hasBackslash && braceEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeBraceEscape(chunkData)
|
||||
} else if perlEscapePat.Match(chunkData) {
|
||||
} else if hasBackslash && perlEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodePerlEscape(chunkData)
|
||||
} else if htmlEscapePat.Match(chunkData) {
|
||||
} else if bytes.Contains(chunkData, htmlAnchor) && htmlEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeHtmlEscape(chunkData)
|
||||
} else if percentEscapePat.Match(chunkData) {
|
||||
} else if bytes.Contains(chunkData, percentAnchor) && percentEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodePercentEscape(chunkData)
|
||||
} else if escapePat.Match(chunkData) {
|
||||
} else if hasBackslash && escapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeEscaped(chunkData)
|
||||
} else if codePointPat.Match(chunkData) {
|
||||
} else if codePointIdx := codePointMatches(chunkData); len(codePointIdx) > 0 {
|
||||
matched = true
|
||||
chunkData = decodeCodePoint(chunkData)
|
||||
} else if cssEscapePat.Match(chunkData) {
|
||||
chunkData = decodeCodePointAt(chunkData, codePointIdx)
|
||||
} else if hasBackslash && cssEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeCssEscape(chunkData)
|
||||
// } else if hexEscapePat.Match(chunkData) {
|
||||
@@ -154,8 +210,13 @@ func decodeWithPattern(input []byte, re *regexp.Regexp) []byte {
|
||||
|
||||
func decodeCodePoint(input []byte) []byte {
|
||||
// Find all Unicode escape sequences in the input byte slice
|
||||
indices := codePointPat.FindAllSubmatchIndex(input, -1)
|
||||
return decodeCodePointAt(input, codePointMatches(input))
|
||||
}
|
||||
|
||||
// decodeCodePointAt is decodeCodePoint driven by indices the caller already
|
||||
// has. FromChunk needs the match list to decide whether this pattern applies,
|
||||
// so it passes that list straight through rather than scanning a second time.
|
||||
func decodeCodePointAt(input []byte, indices [][]int) []byte {
|
||||
// Iterate over found indices in reverse order to avoid modifying the slice length
|
||||
utf8Bytes := make([]byte, maxBytesPerRune)
|
||||
for i := len(indices) - 1; i >= 0; i-- {
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
regexp "github.com/wasilibs/go-re2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
// legacyCodePointPat is main's pattern, word boundary and all.
|
||||
var legacyCodePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)
|
||||
|
||||
// legacyDecodeCodePoint is main's decodeCodePoint verbatim, driven by the
|
||||
// boundary-carrying pattern.
|
||||
func legacyDecodeCodePoint(input []byte) []byte {
|
||||
indices := legacyCodePointPat.FindAllSubmatchIndex(input, -1)
|
||||
utf8Bytes := make([]byte, maxBytesPerRune)
|
||||
for i := len(indices) - 1; i >= 0; i-- {
|
||||
matches := indices[i]
|
||||
startIndex, endIndex := matches[0], matches[1]
|
||||
hexStartIndex, hexEndIndex := matches[2], matches[3]
|
||||
if endIndex != hexEndIndex && input[endIndex-1] != spaceChar {
|
||||
endIndex = endIndex - 1
|
||||
}
|
||||
unicodeInt, err := strconv.ParseInt(string(input[hexStartIndex:hexEndIndex]), 16, 32)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
|
||||
input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
// compareIndices checks that the boundary filter selects exactly the matches
|
||||
// the \b pattern would have found.
|
||||
func compareIndices(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
want := legacyCodePointPat.FindAllSubmatchIndex(data, -1)
|
||||
got := codePointMatches(bytes.Clone(data))
|
||||
if len(want) != len(got) {
|
||||
t.Fatalf("input %q: legacy found %d matches %v, filtered found %d %v", data, len(want), want, len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
for j := range want[i] {
|
||||
if want[i][j] != got[i][j] {
|
||||
t.Fatalf("input %q: match %d differs: legacy %v filtered %v", data, i, want[i], got[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compareDecode checks the full decode output agrees.
|
||||
func compareDecode(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
want := legacyDecodeCodePoint(bytes.Clone(data))
|
||||
got := decodeCodePoint(bytes.Clone(data))
|
||||
if !bytes.Equal(want, got) {
|
||||
t.Fatalf("input %q: legacy decoded to %q, new decoded to %q", data, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// compareMatched checks the decoder's decode/skip decision agrees.
|
||||
func compareMatched(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
wantMatched := legacyCodePointPat.Match(data)
|
||||
gotMatched := len(codePointMatches(bytes.Clone(data))) > 0
|
||||
if wantMatched != gotMatched {
|
||||
t.Fatalf("input %q: legacy matched=%v, new matched=%v", data, wantMatched, gotMatched)
|
||||
}
|
||||
}
|
||||
|
||||
var codePointSeeds = []string{
|
||||
"", "U+0041", "U+0041 ", "U+0041x", "U+0041U+0042", "U+0041 U+0042",
|
||||
"AU+0041", "_U+0041", "9U+0041", " U+0041", "\tU+0041", "\nU+0041",
|
||||
"AU+0041 U+0042", "U+0041\nU+0042", "U+0041\tU+0042", "U+00",
|
||||
"U+ZZZZ", "u+0041", "UU+0041", "U+0041U", "U+0041+", "xU+0041U+0042 U+0043",
|
||||
"U+D800", "U+FFFF", "U+0000", "café U+00E9", "\xff\xfeU+0041",
|
||||
"U+0041U+0042U+0043", "-U+0041-U+0042-", "U+0041.U+0042", "1U+0041 2U+0042",
|
||||
}
|
||||
|
||||
func TestCodePointBoundaryEquivalence(t *testing.T) {
|
||||
for _, s := range codePointSeeds {
|
||||
compareIndices(t, []byte(s))
|
||||
compareDecode(t, []byte(s))
|
||||
compareMatched(t, []byte(s))
|
||||
}
|
||||
|
||||
// Alphabet weighted towards everything the pattern and boundary care about.
|
||||
alphabet := []byte("UU++0123456789abcdefABCDEFxyz_ \t\n.-\xff")
|
||||
rnd := rand.New(rand.NewSource(7))
|
||||
for i := 0; i < 300000; i++ {
|
||||
b := make([]byte, 1+rnd.Intn(40))
|
||||
for j := range b {
|
||||
b[j] = alphabet[rnd.Intn(len(alphabet))]
|
||||
}
|
||||
compareIndices(t, b)
|
||||
compareDecode(t, b)
|
||||
compareMatched(t, b)
|
||||
}
|
||||
}
|
||||
|
||||
// legacyFromChunk is main's whole FromChunk, unanchored and with \b.
|
||||
func legacyFromChunk(chunk *sources.Chunk) []byte {
|
||||
chunkData := bytes.Clone(chunk.Data)
|
||||
matched := false
|
||||
if longEscapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodeLongEscape(chunkData)
|
||||
} else if braceEscapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodeBraceEscape(chunkData)
|
||||
} else if perlEscapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodePerlEscape(chunkData)
|
||||
} else if htmlEscapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodeHtmlEscape(chunkData)
|
||||
} else if percentEscapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodePercentEscape(chunkData)
|
||||
} else if escapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodeEscaped(chunkData)
|
||||
} else if legacyCodePointPat.Match(chunkData) {
|
||||
matched, chunkData = true, legacyDecodeCodePoint(chunkData)
|
||||
} else if cssEscapePat.Match(chunkData) {
|
||||
matched, chunkData = true, decodeCssEscape(chunkData)
|
||||
}
|
||||
if !matched {
|
||||
return nil
|
||||
}
|
||||
return chunkData
|
||||
}
|
||||
|
||||
func compareFromChunk(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
want := legacyFromChunk(&sources.Chunk{Data: bytes.Clone(data)})
|
||||
got := (&EscapedUnicode{}).FromChunk(&sources.Chunk{Data: bytes.Clone(data)})
|
||||
switch {
|
||||
case want == nil && got != nil:
|
||||
t.Fatalf("new decoded %q to %q, legacy skipped it", data, got.Data)
|
||||
case want != nil && got == nil:
|
||||
t.Fatalf("new skipped %q, legacy decoded it to %q", data, want)
|
||||
case want != nil && !bytes.Equal(want, got.Data):
|
||||
t.Fatalf("input %q: legacy %q, new %q", data, want, got.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromChunkEquivalence(t *testing.T) {
|
||||
for _, s := range append(codePointSeeds,
|
||||
"\\u0041", "\\\\u0041", "\\U0001F600", "\\u{1F600}", "\\x{263A}",
|
||||
"A", "%u0041", "\\41 ", "\\41\\", "plain text", "a+b&c%d\\e",
|
||||
) {
|
||||
compareFromChunk(t, []byte(s))
|
||||
}
|
||||
alphabet := []byte(`UU++\\\uUxX{}&#%;0123456789abcdefABCDEF _.-` + "\t\n")
|
||||
rnd := rand.New(rand.NewSource(11))
|
||||
for i := 0; i < 300000; i++ {
|
||||
b := make([]byte, 1+rnd.Intn(48))
|
||||
for j := range b {
|
||||
b[j] = alphabet[rnd.Intn(len(alphabet))]
|
||||
}
|
||||
compareFromChunk(t, b)
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzCodePointBoundary(f *testing.F) {
|
||||
for _, s := range codePointSeeds {
|
||||
f.Add([]byte(s))
|
||||
}
|
||||
f.Add([]byte("\\u0041"))
|
||||
f.Add([]byte("A"))
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
compareIndices(t, data)
|
||||
compareDecode(t, data)
|
||||
compareFromChunk(t, data)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user