Add additional unicode escape support (#4296)
* Update escaped_unicode.go * Create escaped_unicode_bench_test.go * Update escaped_unicode_test.go * updated unicode escape logic * comment updates --------- Co-authored-by: Joe Leon <> Co-authored-by: Shahzad Haider <[email protected]>
This commit is contained in:
jltrfl
co-authored by
Joe Leon <>
Shahzad Haider
Joe Leon <>
Shahzad Haider
parent
c319bb811e
commit
acdd6f846f
+167
-33
@@ -23,6 +23,33 @@ var (
|
||||
|
||||
// Common escape sequence used in programming languages.
|
||||
escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`)
|
||||
|
||||
// Additional Unicode escape formats from dencode.com
|
||||
|
||||
// \u{X} format - Rust, Swift, some JS, etc. (variable length hex in braces)
|
||||
braceEscapePat = regexp.MustCompile(`\\u\{([a-fA-F0-9]{1,6})\}`)
|
||||
|
||||
// \U00XXXXXX format - Python, etc. (8-digit format for non-BMP characters)
|
||||
longEscapePat = regexp.MustCompile(`\\U([a-fA-F0-9]{8})`)
|
||||
|
||||
// \x{X} format - Perl (variable length hex in braces)
|
||||
perlEscapePat = regexp.MustCompile(`\\x\{([a-fA-F0-9]{1,6})\}`)
|
||||
|
||||
// \X format - CSS (hex without padding). Go's regexp (RE2) has no look-ahead, so we
|
||||
// include the delimiter (whitespace, another backslash, or end-of-string) in the
|
||||
// match using a non-capturing group. The delimiter is later re-inserted by the
|
||||
// decoder when necessary.
|
||||
cssEscapePat = regexp.MustCompile(`\\([a-fA-F0-9]{1,6})(?:\s|\\|$)`)
|
||||
|
||||
// &#xX; format - HTML/XML (hex with semicolon)
|
||||
htmlEscapePat = regexp.MustCompile(`&#x([a-fA-F0-9]{1,6});`)
|
||||
|
||||
// %uXXXX format - Percent-encoding (non-standard)
|
||||
percentEscapePat = regexp.MustCompile(`%u([a-fA-F0-9]{4})`)
|
||||
|
||||
// // 0xX format - Hexadecimal notation with space separation
|
||||
// Note: Commenting out for now due to high memory overhead. Review ways to handle this.
|
||||
// hexEscapePat = regexp.MustCompile(`0x([a-fA-F0-9]{1,6})(?:\s|$)`)
|
||||
)
|
||||
|
||||
func (d *EscapedUnicode) Type() detectorspb.DecoderType {
|
||||
@@ -39,13 +66,38 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
chunkData = bytes.Clone(chunk.Data)
|
||||
matched = false
|
||||
)
|
||||
if codePointPat.Match(chunkData) {
|
||||
|
||||
// Process patterns in priority order - more specific patterns first
|
||||
// This prevents conflicts where multiple patterns match the same input
|
||||
|
||||
// Long escape format (8 hex digits) - highest priority
|
||||
if longEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeCodePoint(chunkData)
|
||||
}
|
||||
if escapePat.Match(chunkData) {
|
||||
chunkData = decodeLongEscape(chunkData)
|
||||
} else if braceEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeBraceEscape(chunkData)
|
||||
} else if perlEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodePerlEscape(chunkData)
|
||||
} else if htmlEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeHtmlEscape(chunkData)
|
||||
} else if percentEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodePercentEscape(chunkData)
|
||||
} else if escapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeEscaped(chunkData)
|
||||
} else if codePointPat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeCodePoint(chunkData)
|
||||
} else if cssEscapePat.Match(chunkData) {
|
||||
matched = true
|
||||
chunkData = decodeCssEscape(chunkData)
|
||||
// } else if hexEscapePat.Match(chunkData) {
|
||||
// matched = true
|
||||
// chunkData = decodeHexEscape(chunkData)
|
||||
}
|
||||
|
||||
if matched {
|
||||
@@ -71,6 +123,34 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
const maxBytesPerRune = 4
|
||||
const spaceChar = byte(' ')
|
||||
|
||||
// decodeWithPattern replaces escape sequences matched by re with their UTF-8
|
||||
// equivalents. The regex *must* have the first capturing group contain the
|
||||
// hexadecimal code-point digits. Any invalid value (> 0x10FFFF or parse error)
|
||||
// is skipped. The replacement walks matches in reverse order to avoid index
|
||||
// shifts.
|
||||
func decodeWithPattern(input []byte, re *regexp.Regexp) []byte {
|
||||
indices := re.FindAllSubmatchIndex(input, -1)
|
||||
if len(indices) == 0 {
|
||||
return input
|
||||
}
|
||||
|
||||
utf8Bytes := make([]byte, maxBytesPerRune)
|
||||
for i := len(indices) - 1; i >= 0; i-- {
|
||||
m := indices[i]
|
||||
start, end := m[0], m[1]
|
||||
hexStart, hexEnd := m[2], m[3]
|
||||
|
||||
cp, err := strconv.ParseUint(string(input[hexStart:hexEnd]), 16, 32)
|
||||
if err != nil || cp > 0x10FFFF {
|
||||
continue
|
||||
}
|
||||
|
||||
utf8Len := utf8.EncodeRune(utf8Bytes, rune(cp))
|
||||
input = append(input[:start], append(utf8Bytes[:utf8Len], input[end:]...)...)
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
func decodeCodePoint(input []byte) []byte {
|
||||
// Find all Unicode escape sequences in the input byte slice
|
||||
indices := codePointPat.FindAllSubmatchIndex(input, -1)
|
||||
@@ -112,33 +192,87 @@ func decodeCodePoint(input []byte) []byte {
|
||||
}
|
||||
|
||||
func decodeEscaped(input []byte) []byte {
|
||||
// Find all Unicode escape sequences in the input byte slice
|
||||
indices := escapePat.FindAllSubmatchIndex(input, -1)
|
||||
|
||||
// Iterate over found indices in reverse order to avoid modifying the slice length
|
||||
utf8Bytes := make([]byte, maxBytesPerRune)
|
||||
for i := len(indices) - 1; i >= 0; i-- {
|
||||
matches := indices[i]
|
||||
startIndex := matches[0]
|
||||
hexStartIndex := matches[2]
|
||||
endIndex := matches[3]
|
||||
|
||||
// Extract the hexadecimal value from the escape sequence
|
||||
hexValue := string(input[hexStartIndex:endIndex])
|
||||
|
||||
// Parse the hexadecimal value to an integer
|
||||
unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
|
||||
if err != nil {
|
||||
// If there's an error, continue to the next escape sequence
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert the Unicode code point to a UTF-8 representation
|
||||
utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
|
||||
|
||||
// Replace the escape sequence with the UTF-8 representation
|
||||
input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
|
||||
}
|
||||
|
||||
return input
|
||||
return decodeWithPattern(input, escapePat)
|
||||
}
|
||||
|
||||
// decodeBraceEscape handles \u{X} format - Rust, Swift, some JS, etc.
|
||||
func decodeBraceEscape(input []byte) []byte {
|
||||
return decodeWithPattern(input, braceEscapePat)
|
||||
}
|
||||
|
||||
// decodeLongEscape handles \U00XXXXXX format - Python, etc.
|
||||
func decodeLongEscape(input []byte) []byte {
|
||||
return decodeWithPattern(input, longEscapePat)
|
||||
}
|
||||
|
||||
// decodePerlEscape handles \x{X} format - Perl
|
||||
func decodePerlEscape(input []byte) []byte {
|
||||
return decodeWithPattern(input, perlEscapePat)
|
||||
}
|
||||
|
||||
// decodeCssEscape handles \X format - CSS (hex without padding, with space delimiter or end of string or next hex sequence)
|
||||
func decodeCssEscape(input []byte) []byte {
|
||||
return decodeWithPattern(input, cssEscapePat)
|
||||
}
|
||||
|
||||
// decodeHtmlEscape handles &#xX; format - HTML/XML
|
||||
func decodeHtmlEscape(input []byte) []byte {
|
||||
return decodeWithPattern(input, htmlEscapePat)
|
||||
}
|
||||
|
||||
// decodePercentEscape handles %uXXXX format - Percent-encoding (non-standard)
|
||||
func decodePercentEscape(input []byte) []byte {
|
||||
return decodeWithPattern(input, percentEscapePat)
|
||||
}
|
||||
|
||||
// decodeHexEscape handles 0xX format - Hexadecimal notation with space separation
|
||||
// func decodeHexEscape(input []byte) []byte {
|
||||
// // This format requires consecutive 0xNN sequences to be considered for decoding
|
||||
// // We'll look for patterns of multiple consecutive hex values
|
||||
// hexPattern := regexp.MustCompile(`(?:0x[a-fA-F0-9]{1,2}(?:\s+|$))+`)
|
||||
|
||||
// matches := hexPattern.FindAll(input, -1)
|
||||
// if len(matches) == 0 {
|
||||
// return input
|
||||
// }
|
||||
|
||||
// result := input
|
||||
// for _, match := range matches {
|
||||
// // Extract individual hex values
|
||||
// individualHex := regexp.MustCompile(`0x([a-fA-F0-9]{1,2})`)
|
||||
// hexMatches := individualHex.FindAllSubmatch(match, -1)
|
||||
|
||||
// // Only decode if we have multiple consecutive hex values (likely to be a Unicode string)
|
||||
// if len(hexMatches) < 3 {
|
||||
// continue
|
||||
// }
|
||||
|
||||
// var decoded []byte
|
||||
// for _, hexMatch := range hexMatches {
|
||||
// hexValue := string(hexMatch[1])
|
||||
// if len(hexValue) == 1 {
|
||||
// hexValue = "0" + hexValue // Pad single digit hex values
|
||||
// }
|
||||
|
||||
// unicodeInt, err := strconv.ParseUint(hexValue, 16, 32)
|
||||
// if err != nil || unicodeInt > 0x10FFFF {
|
||||
// break
|
||||
// }
|
||||
|
||||
// if unicodeInt <= 0x7F {
|
||||
// // ASCII character
|
||||
// decoded = append(decoded, byte(unicodeInt))
|
||||
// } else {
|
||||
// // Unicode character
|
||||
// utf8Bytes := make([]byte, maxBytesPerRune)
|
||||
// utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
|
||||
// decoded = append(decoded, utf8Bytes[:utf8Len]...)
|
||||
// }
|
||||
// }
|
||||
|
||||
// // Replace the original sequence with decoded bytes
|
||||
// result = bytes.Replace(result, match, decoded, 1)
|
||||
// }
|
||||
|
||||
// return result
|
||||
// }
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
// Benchmark data for testing
|
||||
var (
|
||||
// Original formats
|
||||
originalUnicodeData = []byte("\\u0041\\u004b\\u0049\\u0041\\u0055\\u004d\\u0034\\u0047\\u0036\\u004f\\u0036\\u004e\\u0041\\u004b\\u0045\\u0037\\u004c\\u0043\\u0044\\u004a")
|
||||
codePointData = []byte("U+0041 U+004B U+0049 U+0041 U+0055 U+004D U+0034 U+0047 U+0036 U+004F U+0036 U+004E U+0041 U+004B U+0045 U+0037 U+004C U+0043 U+0044 U+004A")
|
||||
|
||||
// New formats
|
||||
braceEscapeData = []byte("\\u{41}\\u{4b}\\u{49}\\u{41}\\u{55}\\u{4d}\\u{34}\\u{47}\\u{36}\\u{4f}\\u{36}\\u{4e}\\u{41}\\u{4b}\\u{45}\\u{37}\\u{4c}\\u{43}\\u{44}\\u{4a}")
|
||||
longEscapeData = []byte("\\U00000041\\U0000004b\\U00000049\\U00000041\\U00000055\\U0000004d\\U00000034\\U00000047\\U00000036\\U0000004f\\U00000036\\U0000004e\\U00000041\\U0000004b\\U00000045\\U00000037\\U0000004c\\U00000043\\U00000044\\U0000004a")
|
||||
perlEscapeData = []byte("\\x{41}\\x{4b}\\x{49}\\x{41}\\x{55}\\x{4d}\\x{34}\\x{47}\\x{36}\\x{4f}\\x{36}\\x{4e}\\x{41}\\x{4b}\\x{45}\\x{37}\\x{4c}\\x{43}\\x{44}\\x{4a}")
|
||||
cssEscapeData = []byte("\\41 \\4b \\49 \\41 \\55 \\4d \\34 \\47 \\36 \\4f \\36 \\4e \\41 \\4b \\45 \\37 \\4c \\43 \\44 \\4a ")
|
||||
htmlEscapeData = []byte("AKIAUM4G6O6NAKE7LCDJ")
|
||||
percentEscapeData = []byte("%u0041%u004b%u0049%u0041%u0055%u004d%u0034%u0047%u0036%u004f%u0036%u004e%u0041%u004b%u0045%u0037%u004c%u0043%u0044%u004a")
|
||||
//hexEscapeData = []byte("0x41 0x4b 0x49 0x41 0x55 0x4d 0x34 0x47 0x36 0x4f 0x36 0x4e 0x41 0x4b 0x45 0x37 0x4c 0x43 0x44 0x4a ")
|
||||
|
||||
// Mixed content (more realistic scenario)
|
||||
mixedContentData = []byte(`
|
||||
const config = {
|
||||
apiKey: "\\u0041\\u004b\\u0049\\u0041\\u0055\\u004d\\u0034\\u0047\\u0036\\u004f\\u0036\\u004e\\u0041\\u004b\\u0045\\u0037\\u004c\\u0043\\u0044\\u004a",
|
||||
secretKey: "\\u{6e}\\u{62}\\u{75}\\u{68}\\u{7a}\\u{4b}\\u{79}\\u{39}\\u{50}\\u{50}\\u{7a}\\u{32}\\u{7a}\\u{47}\\u{33}\\u{47}\\u{54}\\u{4a}\\u{71}\\u{4b}\\u{45}\\u{43}\\u{6e}\\u{71}\\u{4c}\\u{41}\\u{78}\\u{43}\\u{76}\\u{2f}\\u{36}\\u{68}\\u{43}\\u{6a}\\u{6b}\\u{50}\\u{68}\\u{66}\\u{58}\\u{6f}",
|
||||
htmlToken: "AKIAUM4G6O6NAKE7LCDJ",
|
||||
normalText: "This is normal text that should not be processed"
|
||||
}
|
||||
`)
|
||||
|
||||
// Large data for stress testing
|
||||
largeData = func() []byte {
|
||||
data := make([]byte, 0, 10000)
|
||||
for i := 0; i < 100; i++ {
|
||||
data = append(data, originalUnicodeData...)
|
||||
data = append(data, braceEscapeData...)
|
||||
data = append(data, longEscapeData...)
|
||||
data = append(data, htmlEscapeData...)
|
||||
data = append(data, []byte(" normal text ")...)
|
||||
}
|
||||
return data
|
||||
}()
|
||||
|
||||
// No Unicode data (worst case for performance)
|
||||
noUnicodeData = []byte(`
|
||||
This is a large block of text with no Unicode escape sequences.
|
||||
It contains various programming constructs like:
|
||||
- Variable declarations: var x = 123;
|
||||
- Function calls: doSomething(param1, param2);
|
||||
- Comments: /* this is a comment */
|
||||
- Strings: "hello world"
|
||||
- Numbers: 42, 3.14159, 0xFF
|
||||
- But no Unicode escapes that would trigger our decoders.
|
||||
This simulates the common case where files don't contain Unicode escapes.
|
||||
`)
|
||||
)
|
||||
|
||||
// Benchmark individual decoder functions
|
||||
func BenchmarkDecodeOriginalEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodeEscaped(originalUnicodeData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeCodePoint(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodeCodePoint(codePointData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBraceEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodeBraceEscape(braceEscapeData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeLongEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodeLongEscape(longEscapeData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodePerlEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodePerlEscape(perlEscapeData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeCssEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodeCssEscape(cssEscapeData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeHtmlEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodeHtmlEscape(htmlEscapeData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodePercentEscape(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decodePercentEscape(percentEscapeData)
|
||||
}
|
||||
}
|
||||
|
||||
// func BenchmarkDecodeHexEscape(b *testing.B) {
|
||||
// for i := 0; i < b.N; i++ {
|
||||
// _ = decodeHexEscape(hexEscapeData)
|
||||
// }
|
||||
// }
|
||||
|
||||
// Benchmark the full FromChunk method with different data types
|
||||
func BenchmarkFromChunk_OriginalFormat(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: originalUnicodeData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunk_BraceFormat(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: braceEscapeData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunk_LongFormat(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: longEscapeData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunk_HtmlFormat(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: htmlEscapeData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunk_MixedContent(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: mixedContentData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunk_NoUnicode(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: noUnicodeData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunk_LargeData(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: largeData}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = decoder.FromChunk(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark regex matching performance (most expensive operation)
|
||||
func BenchmarkRegexMatching_AllPatterns(b *testing.B) {
|
||||
testData := mixedContentData
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Simulate the pattern matching in FromChunk
|
||||
_ = longEscapePat.Match(testData)
|
||||
_ = braceEscapePat.Match(testData)
|
||||
_ = perlEscapePat.Match(testData)
|
||||
_ = htmlEscapePat.Match(testData)
|
||||
_ = percentEscapePat.Match(testData)
|
||||
_ = escapePat.Match(testData)
|
||||
_ = codePointPat.Match(testData)
|
||||
_ = cssEscapePat.Match(testData)
|
||||
//_ = hexEscapePat.Match(testData)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRegexMatching_NoMatch(b *testing.B) {
|
||||
testData := noUnicodeData
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Simulate the pattern matching in FromChunk on data with no matches
|
||||
_ = longEscapePat.Match(testData)
|
||||
_ = braceEscapePat.Match(testData)
|
||||
_ = perlEscapePat.Match(testData)
|
||||
_ = htmlEscapePat.Match(testData)
|
||||
_ = percentEscapePat.Match(testData)
|
||||
_ = escapePat.Match(testData)
|
||||
_ = codePointPat.Match(testData)
|
||||
_ = cssEscapePat.Match(testData)
|
||||
//_ = hexEscapePat.Match(testData)
|
||||
}
|
||||
}
|
||||
|
||||
// Memory allocation benchmarks
|
||||
func BenchmarkFromChunk_MemoryAllocation(b *testing.B) {
|
||||
decoder := &EscapedUnicode{}
|
||||
chunk := &sources.Chunk{Data: mixedContentData}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
result := decoder.FromChunk(chunk)
|
||||
if result != nil {
|
||||
// Prevent compiler optimization
|
||||
_ = result.Data
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,139 @@ func TestUnicodeEscape_FromChunk(t *testing.T) {
|
||||
},
|
||||
},
|
||||
|
||||
// New test cases for additional Unicode escape formats
|
||||
|
||||
// \u{X} format - Rust, Swift, some JS, etc.
|
||||
{
|
||||
name: "[brace] \\u{X} format - Rust/Swift style",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("\\u{74}\\u{6f}\\u{6b}\\u{65}\\u{6e}\\u{3a}\\u{20}\\u{22}\\u{67}\\u{68}\\u{70}\\u{5f}\\u{49}\\u{77}\\u{64}\\u{4d}\\u{78}\\u{39}\\u{57}\\u{46}\\u{57}\\u{52}\\u{52}\\u{66}\\u{4d}\\u{68}\\u{54}\\u{59}\\u{69}\\u{61}\\u{56}\\u{6a}\\u{5a}\\u{37}\\u{38}\\u{4a}\\u{66}\\u{75}\\u{61}\\u{6d}\\u{76}\\u{6e}\\u{30}\\u{59}\\u{57}\\u{52}\\u{4d}\\u{30}\\u{22}"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("token: \"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\""),
|
||||
},
|
||||
},
|
||||
|
||||
// \U00XXXXXX format - Python, etc.
|
||||
{
|
||||
name: "[long] \\U00XXXXXX format - Python style",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("\\U00000074\\U0000006f\\U0000006b\\U00000065\\U0000006e\\U0000003a\\U00000020\\U00000022\\U00000067\\U00000068\\U00000070\\U0000005f\\U00000049\\U00000077\\U00000064\\U0000004d\\U00000078\\U00000039\\U00000057\\U00000046\\U00000057\\U00000052\\U00000052\\U00000066\\U0000004d\\U00000068\\U00000054\\U00000059\\U00000069\\U00000061\\U00000056\\U0000006a\\U0000005a\\U00000037\\U00000038\\U0000004a\\U00000066\\U00000075\\U00000061\\U0000006d\\U00000076\\U0000006e\\U00000030\\U00000059\\U00000057\\U00000052\\U0000004d\\U00000030\\U00000022"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("token: \"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\""),
|
||||
},
|
||||
},
|
||||
|
||||
// \x{X} format - Perl
|
||||
{
|
||||
name: "[perl] \\x{X} format - Perl style",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("\\x{74}\\x{6f}\\x{6b}\\x{65}\\x{6e}\\x{3a}\\x{20}\\x{22}\\x{67}\\x{68}\\x{70}\\x{5f}\\x{49}\\x{77}\\x{64}\\x{4d}\\x{78}\\x{39}\\x{57}\\x{46}\\x{57}\\x{52}\\x{52}\\x{66}\\x{4d}\\x{68}\\x{54}\\x{59}\\x{69}\\x{61}\\x{56}\\x{6a}\\x{5a}\\x{37}\\x{38}\\x{4a}\\x{66}\\x{75}\\x{61}\\x{6d}\\x{76}\\x{6e}\\x{30}\\x{59}\\x{57}\\x{52}\\x{4d}\\x{30}\\x{22}"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("token: \"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\""),
|
||||
},
|
||||
},
|
||||
|
||||
// \X format - CSS (space delimited)
|
||||
// ToDo: Look into supporting CSS where there is no whitespace ex: \013322\013171\013001. Currently not supported by this implementation.
|
||||
{
|
||||
name: "[css] \\X format - CSS style",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("\\74 \\6f \\6b \\65 \\6e \\3a \\20 \\22 \\67 \\68 \\70 \\5f \\49 \\77 \\64 \\4d \\78 \\39 \\57 \\46 \\57 \\52 \\52 \\66 \\4d \\68 \\54 \\59 \\69 \\61 \\56 \\6a \\5a \\37 \\38 \\4a \\66 \\75 \\61 \\6d \\76 \\6e \\30 \\59 \\57 \\52 \\4d \\30 \\22 "),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("token: \"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\""),
|
||||
},
|
||||
},
|
||||
|
||||
// &#xX; format - HTML/XML
|
||||
{
|
||||
name: "[html] &#xX; format - HTML/XML style",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("token: "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""),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("token: \"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\""),
|
||||
},
|
||||
},
|
||||
|
||||
// %uXXXX format - Percent-encoding (non-standard)
|
||||
{
|
||||
name: "[percent] %uXXXX format - Percent encoding",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("%u0074%u006f%u006b%u0065%u006e%u003a%u0020%u0022%u0067%u0068%u0070%u005f%u0049%u0077%u0064%u004d%u0078%u0039%u0057%u0046%u0057%u0052%u0052%u0066%u004d%u0068%u0054%u0059%u0069%u0061%u0056%u006a%u005a%u0037%u0038%u004a%u0066%u0075%u0061%u006d%u0076%u006e%u0030%u0059%u0057%u0052%u004d%u0030%u0022"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("token: \"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\""),
|
||||
},
|
||||
},
|
||||
|
||||
// // 0xX format - Hexadecimal notation with space separation
|
||||
// {
|
||||
// name: "[hex] 0xX format - Hex with spaces",
|
||||
// chunk: &sources.Chunk{
|
||||
// Data: []byte("0x74 0x6f 0x6b 0x65 0x6e 0x3a 0x20 0x22 0x67 0x68 0x70 0x5f 0x49 0x77 0x64 0x4d 0x78 0x39 0x57 0x46 0x57 0x52 0x52 0x66 0x4d 0x68 0x54 0x59 0x69 0x61 0x56 0x6a 0x5a 0x37 0x38 0x4a 0x66 0x75 0x61 0x6d 0x76 0x6e 0x30 0x59 0x57 0x52 0x4d 0x30 0x22 "),
|
||||
// },
|
||||
// want: &sources.Chunk{
|
||||
// Data: []byte("token: \"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\""),
|
||||
// },
|
||||
// },
|
||||
|
||||
// // 0xX format - Hexadecimal notation with comma separation
|
||||
// {
|
||||
// name: "[hex] 0xX format - Hex with commas",
|
||||
// chunk: &sources.Chunk{
|
||||
// Data: []byte("0x74,0x6f,0x6b,0x65,0x6e,0x3a,0x20,0x22,0x67,0x68,0x70,0x5f,0x49,0x77,0x64,0x4d,0x78,0x39,0x57,0x46,0x57,0x52,0x52,0x66,0x4d,0x68,0x54,0x59,0x69,0x61,0x56,0x6a,0x5a,0x37,0x38,0x4a,0x66,0x75,0x61,0x6d,0x76,0x6e,0x30,0x59,0x57,0x52,0x4d,0x30,0x22"),
|
||||
// },
|
||||
// want: &sources.Chunk{
|
||||
// Data: []byte("token: \"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\""),
|
||||
// },
|
||||
// },
|
||||
|
||||
// Test cases for mixed content with new formats
|
||||
{
|
||||
name: "[mixed] \\u{X} in code context",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("const secret = \"\\u{41}\\u{4b}\\u{49}\\u{41}\\u{55}\\u{4d}\\u{34}\\u{47}\\u{36}\\u{4f}\\u{36}\\u{4e}\\u{41}\\u{4b}\\u{45}\\u{37}\\u{4c}\\u{43}\\u{44}\\u{4a}\";"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("const secret = \"AKIAUM4G6O6NAKE7LCDJ\";"),
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "[mixed] HTML entity in web context",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("<span>AWS Key: AKIAUM4G6O6NAKE7LCDJ</span>"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("<span>AWS Key: AKIAUM4G6O6NAKE7LCDJ</span>"),
|
||||
},
|
||||
},
|
||||
|
||||
// Test cases for higher Unicode values (non-BMP)
|
||||
{
|
||||
name: "[emoji] \\u{X} with emoji",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("\\u{1f600} Happy face emoji"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("😀 Happy face emoji"),
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "[emoji] \\U00XXXXXX with emoji",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("\\U0001f600 Happy face emoji"),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte("😀 Happy face emoji"),
|
||||
},
|
||||
},
|
||||
|
||||
// nothing
|
||||
{
|
||||
name: "no escaped",
|
||||
|
||||
Reference in New Issue
Block a user