Brad Larsen
c563a0692f
This adds a new input source to TruffleHog, accessible via `trufflehog json-enumerator`.
This input source requires a list of filenames, each of which is an NDJSON-formatted sequence of objects that take one of two forms:
Form 1: `{"data": "utf-8 string", "metadata": <non-null JSON value>}`
Form 2: `{"data_b64": "base64-encoded bytestring", "metadata": <non-null JSON value>}`
The `data` / `data_b64` field specifies the content to be scanned. The `metadata` field is arbitrary, and is simply propagated downstream with scan results from the corresponding content.
Note that although `trufflehog json-enumerator` requires a list of filenames to be given, the NDJSON data that you wish to scan may not need to be first written to disk. On Linux and macOS, at least, you can use shell process substitution to set up a named pipe from a producer process, like `trufflehog json-enumerator <(some-program-that-emits-ndjson)`.
130 lines
3.0 KiB
Go
130 lines
3.0 KiB
Go
package json_enumerator
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"unicode/utf8"
|
|
|
|
"github.com/go-logr/logr"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
|
)
|
|
|
|
const secretPart1 string = "SECRET"
|
|
const secretPart2 string = "SPLIT"
|
|
|
|
// Split the secret into two parts and pad the rest of the chunk with A's.
|
|
func makeStringData(t *testing.T, chunkSize int) []byte {
|
|
t.Helper()
|
|
data := []byte(strings.Join([]string{
|
|
strings.Repeat("A", chunkSize-len(secretPart1)),
|
|
secretPart1,
|
|
secretPart2,
|
|
strings.Repeat("A", chunkSize-len(secretPart2)),
|
|
}, ""))
|
|
assert.True(t, utf8.Valid(data))
|
|
return data
|
|
}
|
|
|
|
// Split the secret into two parts and pad the rest of the chunk with invalid unicode
|
|
func makeBase64Data(t *testing.T, chunkSize int) []byte {
|
|
t.Helper()
|
|
data := []byte(strings.Join([]string{
|
|
strings.Repeat("\xff", chunkSize-len(secretPart1)),
|
|
secretPart1,
|
|
secretPart2,
|
|
strings.Repeat("\xf0", chunkSize-len(secretPart2)),
|
|
}, ""))
|
|
assert.False(t, utf8.Valid(data))
|
|
return data
|
|
}
|
|
|
|
func makeRawMessage(t *testing.T, payload string) json.RawMessage {
|
|
t.Helper()
|
|
var m json.RawMessage
|
|
require.NoError(t, json.Unmarshal([]byte(payload), &m))
|
|
return m
|
|
}
|
|
|
|
func TestScanEnumerator(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
testCases := []struct {
|
|
name string
|
|
data []byte
|
|
metadata json.RawMessage
|
|
shouldFail bool
|
|
}{
|
|
{
|
|
name: "bad metadata 1",
|
|
data: makeStringData(t, 30),
|
|
metadata: makeRawMessage(t, "null"),
|
|
shouldFail: true,
|
|
},
|
|
{
|
|
name: "small string",
|
|
data: makeStringData(t, 30),
|
|
metadata: makeRawMessage(t, "{}"),
|
|
},
|
|
{
|
|
name: "small bytestring",
|
|
data: makeBase64Data(t, 30),
|
|
metadata: makeRawMessage(t, "{}"),
|
|
},
|
|
{
|
|
name: "big string",
|
|
data: makeStringData(t, sources.DefaultChunkSize*10),
|
|
metadata: makeRawMessage(t, "{}"),
|
|
},
|
|
{
|
|
name: "big bytestring",
|
|
data: makeBase64Data(t, sources.DefaultChunkSize*10),
|
|
metadata: makeRawMessage(t, "{}"),
|
|
},
|
|
}
|
|
|
|
for _, testCase := range testCases {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
readJSON, writeJSON := io.Pipe()
|
|
|
|
chunksChan := make(chan *sources.Chunk, 2)
|
|
var workerError error
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
defer close(chunksChan)
|
|
ctx := context.WithLogger(t.Context(), logr.Discard())
|
|
source := &Source{}
|
|
workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, chunksChan)
|
|
}()
|
|
|
|
enc := json.NewEncoder(writeJSON)
|
|
require.NoError(t, enc.Encode(&jsonEntry{
|
|
Data: testCase.data,
|
|
Metadata: testCase.metadata,
|
|
}))
|
|
require.NoError(t, writeJSON.Close())
|
|
|
|
foundSecret := ""
|
|
for chunk := range chunksChan {
|
|
foundSecret += string(chunk.Data)
|
|
}
|
|
|
|
wg.Wait()
|
|
if testCase.shouldFail {
|
|
require.Error(t, workerError)
|
|
} else {
|
|
require.NoError(t, workerError)
|
|
assert.Contains(t, foundSecret, secretPart1+secretPart2)
|
|
}
|
|
})
|
|
}
|
|
}
|