AKIA1234567890ABCDEF
`)}, want: "AKIA1234567890ABCDEF", }, { // Confluence/Teams encode '&' as '&'. The HTML parser // automatically unescapes entities so detector regexes can match. name: "HTML entities decoded", chunk: &sources.Chunk{Data: []byte(`key=abc&secret=hunter2
`)}, want: "key=abc&secret=hunter2", }, // --- Attribute value extraction --- { // Secrets in href URLs (e.g. tokens in query params). name: "attribute value extraction - href", chunk: &sources.Chunk{Data: []byte(`link`)}, want: "https://api.example.com?token=sk-live-1234\nlink", }, { // Secrets in src URLs (e.g. image CDN tokens). name: "attribute value extraction - src", chunk: &sources.Chunk{Data: []byte(`
`)},
want: "https://cdn.example.com/img?key=secret123",
},
{
// Percent-encoded characters in attribute values (%2D -> '-',
// %5F -> '_') are decoded so detectors see the actual secret.
name: "URL-encoded attribute values decoded",
chunk: &sources.Chunk{Data: []byte(`docs`)},
want: "https://api.example.com?token=sk-live_1234\ndocs",
},
{
// data-* attributes are often used for JS-consumed config values.
name: "data-* attributes extracted",
chunk: &sources.Chunk{Data: []byte(`text
`)}, want: "text\nbody { background: url(\"https://cdn.com?key=secret\"); }", }, { // Script following an inline element must NOT concatenate with // the preceding text; it needs its own newline boundary. name: "script adjacent to inline text gets boundary", chunk: &sources.Chunk{Data: []byte(`text`)}, want: "text\nvar key=\"secret\";", }, { // Style following an inline element must NOT concatenate. name: "style adjacent to inline text gets boundary", chunk: &sources.Chunk{Data: []byte(`text`)}, want: "text\n.x { color: red; }", }, { // Entity-like sequences in script content are raw text and must // NOT be decoded by the residual entity replacer. name: "entities in script preserved as raw text", chunk: &sources.Chunk{Data: []byte(``)}, want: `var url = "a=1&b=2";`, }, { // Entity-like sequences in style content are raw text. name: "entities in style preserved as raw text", chunk: &sources.Chunk{Data: []byte(``)}, want: `body::after { content: "©"; }`, }, { // HTML comments are a common place for debug credentials and // TODO notes with hardcoded passwords. name: "HTML comment content included", chunk: &sources.Chunk{Data: []byte(`visible
`)}, want: "visible\nTODO: remove hardcoded password=hunter2", }, // --- Code and pre blocks --- { /// content is preserved verbatim; these blocks are a
// top location for pasted credentials and key exports.
name: "code/pre blocks preserved",
chunk: &sources.Chunk{Data: []byte(`export AWS_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
`)},
want: "export AWS_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
{
// Multi-line PEM private keys in blocks with
line breaks
// are reconstructed with proper newlines for detector matching.
name: "private key in pre block",
chunk: &sources.Chunk{Data: []byte(`-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA04up8h
-----END RSA PRIVATE KEY-----
`)},
want: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA04up8h\n-----END RSA PRIVATE KEY-----",
},
// --- Whitespace and token boundaries ---
{
// Block elements () produce newline boundaries so adjacent
// paragraphs don't merge tokens.
name: "block elements produce newlines",
chunk: &sources.Chunk{Data: []byte(`
first
second
`)},
want: "first\nsecond",
},
{
// All
variants produce newlines.
name: "br tags produce newlines",
chunk: &sources.Chunk{Data: []byte(`line1
line2
line3
`)},
want: "line1\nline2\nline3",
},
{
// Nested inline elements (, ) do not break the token;
// text flows continuously so "token=sk-live-abc123" stays intact.
name: "nested inline elements preserve text continuity",
chunk: &sources.Chunk{Data: []byte(`token=sk-live-abc123
`)},
want: "token=sk-live-abc123",
},
{
// elements are block-level: each cell gets its own line,
// keeping key/value pairs from merging.
name: "table with secrets",
chunk: &sources.Chunk{Data: []byte(
`API Key AKIAIOSFODNN7EXAMPLE Secret wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
`,
)},
want: "API Key\nAKIAIOSFODNN7EXAMPLE\nSecret\nwJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
{
// Even without wrappers, still inserts block boundaries.
name: "td cells without enclosing tr still get block boundaries",
chunk: &sources.Chunk{Data: []byte(
`key value
`,
)},
want: "key\nvalue",
},
{
// elements produce separate lines.
name: "list items produce separate lines",
chunk: &sources.Chunk{Data: []byte(
`- token: abc123
- secret: def456
`,
)},
want: "token: abc123\nsecret: def456",
},
// --- Real-world source formats ---
{
// Confluence storage format: secrets split across tags,
// an AWS key in plain text, and an href with a URL. Exercises text
// node concatenation, attribute extraction, and block boundaries
// together.
name: "confluence storage format - real world",
chunk: &sources.Chunk{Data: []byte(
`Our API credentials:
` +
`Key: AKIAIOSFODNN7EXAMPLE
` +
`Secret: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
` +
`See AWS Console
`,
)},
want: "Our API credentials:\nKey: AKIAIOSFODNN7EXAMPLE\nSecret: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\nSee\nhttps://console.aws.amazon.com\nAWS Console",
},
{
// Teams message HTML: nested wrappers around tags
// containing a GitHub PAT. Verifies that redundant block wrappers
// collapse to clean newlines.
name: "teams message HTML - real world",
chunk: &sources.Chunk{Data: []byte(
`
` +
`Here is the token for the staging env:
` +
`ghp_ABCDEFghijklmnop1234567890abcde
` +
``,
)},
want: "Here is the token for the staging env:\nghp_ABCDEFghijklmnop1234567890abcde",
},
// --- Syntax highlight boundary detection ---
{
// Teams renders code blocks as adjacent elements within a
// single , using highlight.js classes for syntax coloring.
// Newlines from the original code are lost. The decoder detects
// hljs-* classes and inserts newlines at those boundaries while
// still concatenating non-hljs sibling spans (preserving
// mid-token color splits like the value below split across 3 spans).
name: "teams code block with hljs syntax highlighting",
chunk: &sources.Chunk{Data: []byte(
`
` +
`[header]` +
`key_one` +
` = FIRST_VALUE_ABCDEFGH` +
`key_two` +
` = SECOND_VAL_PART_` +
`X` +
`_END_OF_VALUE` +
`format` +
` = json` +
`
`,
)},
want: "[header]\nkey_one = FIRST_VALUE_ABCDEFGH\nkey_two = SECOND_VAL_PART_X_END_OF_VALUE\nformat = json",
},
{
// Spans without hljs classes must still concatenate, preserving
// the existing split-secret behavior even when hljs spans are
// present elsewhere in the document.
name: "non-hljs sibling spans still concatenate",
chunk: &sources.Chunk{Data: []byte(
`SECRET_FIRST_HALF_1234
`,
)},
want: "SECRET_FIRST_HALF_1234",
},
{
// Various hljs-* class names (not just hljs-function) should
// all trigger line boundaries.
name: "multiple hljs class variants trigger boundaries",
chunk: &sources.Chunk{Data: []byte(
`` +
`const` +
` x = ` +
`"value_one"` +
`const` +
` y = ` +
`"value_two"` +
`
`,
)},
want: "const x =\n\"value_one\"\nconst y =\n\"value_two\"",
},
// --- Zero-width / invisible character stripping ---
{
// Zero-width spaces inserted between characters by rich text editors
// are stripped so detector regexes can match the full token.
name: "zero-width space stripped from secret",
chunk: &sources.Chunk{Data: []byte("TOKEN_\u200BABCDEF_1234
")},
want: "TOKEN_ABCDEF_1234",
},
{
// Multiple invisible codepoint types mixed into a single token.
name: "multiple invisible character types stripped",
chunk: &sources.Chunk{Data: []byte("SECRET\u200C_VALUE\u00AD_HERE\u2060_END\uFEFF
")},
want: "SECRET_VALUE_HERE_END",
},
// --- SVG xlink:href attribute extraction ---
{
// SVG elements use xlink:href for URLs which may contain tokens.
name: "xlink:href extracted from SVG element",
chunk: &sources.Chunk{Data: []byte(``)},
want: "https://api.example.com?token=secret_value_123\nicon",
},
// --- Double-encoded HTML entity decoding ---
{
// Content double-encoded as & becomes & after the parser's
// first pass; the residual entity replacer decodes it to &.
name: "double-encoded ampersand decoded",
chunk: &sources.Chunk{Data: []byte(`key=abc&secret=val
`)},
want: "key=abc&secret=val",
},
{
// Single-encoded entities are handled by the parser; verify the
// residual replacer does not corrupt already-decoded content.
name: "single-encoded entities not double-decoded",
chunk: &sources.Chunk{Data: []byte(`5 > 3 & 2 < 4
`)},
want: "5 > 3 & 2 < 4",
},
// --- ASPX / namespace-prefixed tag support ---
{
// Fragment ASPX pages (master-page-based) have no wrapper;
// they start with an <%@ Page %> directive and use
// as their root. The updated htmlTagPattern must match the
// namespace-prefixed tag so the decoder doesn't return nil.
// The <%@ ... %> directive is emitted as text by the HTML5 parser
// (< followed by % is not a valid tag start, so < is treated literally).
// Namespace element attributes (PlaceHolderMain, server) are also
// emitted because is a namespace-prefixed element.
name: "aspx fragment with namespace tags and no html wrapper",
chunk: &sources.Chunk{Data: []byte(
"<%@ Page MasterPageFile=\"~masterurl/default.master\" %>\n" +
`` +
"api_key=abc123" +
" ",
)},
want: "<%@ Page MasterPageFile=\"~masterurl/default.master\" %>\nPlaceHolderMain\nserver\napi_key=abc123",
},
{
// ASP.NET server controls carry secrets in PascalCase attributes
// (ConnectionString, Text, SelectCommand, etc.) that are not in
// highSignalAttrs and do not use the data- prefix. Namespace-prefixed
// elements must have all their attributes emitted.
name: "aspx server control attribute extraction - ConnectionString",
chunk: &sources.Chunk{Data: []byte(`api_key=secret123
`,
},
{
// SharePoint stores page content entity-encoded inside mso:CanvasContent1.
// The first html.Parse pass decodes the entities to a text string containing
// literal HTML markup. The engine's iterativeDecode then re-applies the HTML
// decoder at depth 2 to extract content from that inner markup.
// At depth 1 (this test), the entity-encoded text is emitted as plain text
// after residualEntityReplacer runs. The msdt:dt="string" attribute is also
// emitted because mso:CanvasContent1 is a namespace-prefixed element.
name: "sharepoint aspx mso:CanvasContent1 secret in nested html",
chunk: &sources.Chunk{Data: []byte(
`` +
`` +
`` +
`<div><p>-----BEGIN OPENSSH PRIVATE KEY-----</p></div>` +
` ` +
` `,
)},
want: "string\n-----BEGIN OPENSSH PRIVATE KEY-----
",
},
// --- Integration: all extraction types in one chunk ---
{
// Combines text nodes (split across spans), URL-decoded attribute
// values, inline script content, and an HTML comment -- all in a
// single chunk. Verifies the decoder handles the full extraction
// surface simultaneously.
name: "mixed content with all extraction types",
chunk: &sources.Chunk{Data: []byte(
`API key: AKIA1234567890ABCDEF
` +
`See docs
` +
`` +
``,
)},
want: "API key: AKIA1234567890ABCDEF\nSee\nhttps://api.example.com?token=sk-live_1234\ndocs\nvar secret = \"ghp_abc123def456\";\nTODO: remove hardcoded password=hunter2",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
feature.HTMLDecoderEnabled.Store(true)
defer feature.HTMLDecoderEnabled.Store(false)
d := &HTML{}
got := d.FromChunk(tt.chunk)
if tt.wantNil {
if got != nil {
t.Errorf("FromChunk() = %q, want nil", string(got.Data))
}
return
}
if got == nil {
t.Fatalf("FromChunk() returned nil, want %q", tt.want)
}
if got.DecoderType != detectorspb.DecoderType_HTML {
t.Errorf("DecoderType = %v, want %v", got.DecoderType, detectorspb.DecoderType_HTML)
}
if string(got.Data) != tt.want {
t.Errorf("FromChunk() data =\n%q\nwant:\n%q", string(got.Data), tt.want)
}
})
}
}
// TestHTML_FeatureFlagDisabled verifies that the decoder is a no-op when
// feature.HTMLDecoderEnabled is false.
func TestHTML_FeatureFlagDisabled(t *testing.T) {
feature.HTMLDecoderEnabled.Store(false)
d := &HTML{}
chunk := &sources.Chunk{Data: []byte(`secret: hunter2
`)}
if got := d.FromChunk(chunk); got != nil {
t.Errorf("FromChunk() should return nil when disabled, got %q", string(got.Data))
}
}
// TestHTML_FeatureFlagEnabled verifies that the decoder processes HTML normally
// when feature.HTMLDecoderEnabled is true.
func TestHTML_FeatureFlagEnabled(t *testing.T) {
feature.HTMLDecoderEnabled.Store(true)
defer feature.HTMLDecoderEnabled.Store(false)
d := &HTML{}
chunk := &sources.Chunk{Data: []byte(`secret: hunter2
`)}
got := d.FromChunk(chunk)
if got == nil {
t.Fatal("FromChunk() returned nil, want decoded chunk")
}
if string(got.Data) != "secret: hunter2" {
t.Errorf("FromChunk() data = %q, want %q", string(got.Data), "secret: hunter2")
}
}
// TestLooksLikeHTML verifies the fast heuristic that decides whether chunk data
// is worth parsing as HTML. It must accept valid HTML tags (including self-closing
// and attribute-bearing) while rejecting plain text, arithmetic comparisons, and
// bare HTML entities -- all of which could appear in non-HTML source content.
func TestLooksLikeHTML(t *testing.T) {
tests := []struct {
name string
data string
want bool
}{
{"simple tag", "hello
", true},
{"self-closing", "
", true},
{"with attributes", ``, true},
{"plain text", "no html here", false},
{"angle brackets but not HTML", "5 < 10 and 20 > 15", false},
{"XML-like", "content ", true},
{"just less-than", "a < b", false},
{"html entity only", "& <", false},
// ASPX / namespace-prefixed tags (literal form)
{"aspx namespace tag with space", ``, true},
{"mso namespace tag with space", ``, true},
{"multi-segment namespace tag", ``, true},
// Entity-encoded HTML tags (positive)
{"entity-encoded div with space", `<div class="foo">`, true},
{"entity-encoded p with gt terminator", `<p>hello</p>`, true},
{"entity-encoded self-closing br", `<br/>`, true},
{"entity-encoded namespace tag", `<asp:Content runat="server">`, true},
// Entity-encoded false positives that must NOT match
{"comparison operator entity-encoded", `x < maxRetries exceeded`, false},
{"comparison operator single word", `result < threshold`, false},
{"template placeholder entity-encoded", `<YOUR_API_KEY>`, false},
{"double-encoded entity not matched", `<div>`, false},
{"json with digit after lt", `{"lt": "<3"}`, false},
{"sql comparison bare lt", `SELECT * FROM t WHERE a < b`, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := looksLikeHTML([]byte(tt.data)); got != tt.want {
t.Errorf("looksLikeHTML(%q) = %v, want %v", tt.data, got, tt.want)
}
})
}
}