Brad Larsen
c563a0692f
This adds a new input source to TruffleHog, accessible via `trufflehog json-enumerator`.
This input source requires a list of filenames, each of which is an NDJSON-formatted sequence of objects that take one of two forms:
Form 1: `{"data": "utf-8 string", "metadata": <non-null JSON value>}`
Form 2: `{"data_b64": "base64-encoded bytestring", "metadata": <non-null JSON value>}`
The `data` / `data_b64` field specifies the content to be scanned. The `metadata` field is arbitrary, and is simply propagated downstream with scan results from the corresponding content.
Note that although `trufflehog json-enumerator` requires a list of filenames to be given, the NDJSON data that you wish to scan may not need to be first written to disk. On Linux and macOS, at least, you can use shell process substitution to set up a named pipe from a producer process, like `trufflehog json-enumerator <(some-program-that-emits-ndjson)`.
44 lines
1.3 KiB
Go
44 lines
1.3 KiB
Go
package engine
|
|
|
|
import (
|
|
"runtime"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
"google.golang.org/protobuf/types/known/anypb"
|
|
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/json_enumerator"
|
|
)
|
|
|
|
// ScanJSONEnumeratorInput scans input that is in JSON Enumerator format
|
|
func (e *Engine) ScanJSONEnumeratorInput(
|
|
ctx context.Context,
|
|
c sources.JSONEnumeratorConfig,
|
|
) (sources.JobProgressRef, error) {
|
|
connection := &sourcespb.JSONEnumerator{
|
|
Paths: c.Paths,
|
|
}
|
|
var conn anypb.Any
|
|
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
|
if err != nil {
|
|
ctx.Logger().Error(err, "failed to marshal JSON enumerator connection")
|
|
return sources.JobProgressRef{}, err
|
|
}
|
|
|
|
sourceName := "trufflehog - JSON enumerator"
|
|
sourceID, jobID, err := e.sourceManager.GetIDs(ctx, sourceName, json_enumerator.SourceType)
|
|
if err != nil {
|
|
ctx.Logger().Error(err, "failed to get IDs from source manager")
|
|
return sources.JobProgressRef{}, err
|
|
}
|
|
|
|
source := &json_enumerator.Source{}
|
|
err = source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU())
|
|
if err != nil {
|
|
return sources.JobProgressRef{}, err
|
|
}
|
|
return e.sourceManager.EnumerateAndScan(ctx, sourceName, source)
|
|
}
|