diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index 6d96c0a0b..db8e81b26 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -329,10 +329,13 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu // scanCollection scans a collection and all its items, folders, and requests. // locally scoped Metadata is updated as we drill down into the collection. func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, collection Collection) { - ctx.Logger().V(2).Info("starting to scan collection", + ctx = context.WithValues(ctx, "collection_name", collection.Info.Name, "collection_uuid", collection.Info.Uid, - "variable_count", len(collection.Variables)) + ) + ctx.Logger().V(2).Info("starting to scan collection", + "variable_count", len(collection.Variables), + ) metadata.CollectionInfo = collection.Info metadata.Type = COLLECTION_TYPE s.attemptToAddKeyword(collection.Info.Name) @@ -354,7 +357,6 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch s.scanAuth(ctx, chunksChan, metadata, collection.Auth, URL{}) ctx.Logger().V(3).Info("Scanning events in collection", - "collection_uid", collection.Info.Uid, "event_count", len(collection.Events), ) for _, event := range collection.Events { @@ -362,24 +364,27 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch } ctx.Logger().V(3).Info("Scanning items in collection", - "collection_uid", collection.Info.Uid, "item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items), ) for _, item := range collection.Items { - s.scanItem(ctx, chunksChan, collection, metadata, item, "") + seenItemIds := make(map[string]struct{}) + s.scanItem(ctx, chunksChan, collection, metadata, item, "", seenItemIds) } - } -func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string) { +func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string, seenItemIds map[string]struct{}) { + ctx = context.WithValue(ctx, "item_uid", item.Uid) + ctx.Logger().V(3).Info("Starting to scan item", - "item_uid", item.Uid, "item_parent_item_id", parentItemId, "item_descendent_item_uids", fp.Map(func(i Item) string { return i.Uid })(item.Items), "item_event_count", len(item.Events), "item_response_count", len(item.Response), "item_variable_count", len(item.Variable), ) + + seenItemIds[item.Uid] = struct{}{} + s.attemptToAddKeyword(item.Name) // override the base collection metadata with item-specific metadata @@ -397,7 +402,13 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c } // recurse through the folders for _, subItem := range item.Items { - s.scanItem(ctx, chunksChan, collection, metadata, subItem, item.Uid) + if _, ok := seenItemIds[subItem.Uid]; ok { + ctx.Logger().Info("Skipping already-seen item", + "seen_item_id", subItem.Uid, + ) + continue + } + s.scanItem(ctx, chunksChan, collection, metadata, subItem, item.Uid, seenItemIds) } // The assignment of the folder ID to be the current item UID is due to wanting to assume that your current item is a folder unless you have request data inside of your item. @@ -463,7 +474,8 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT } - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, data, DefaultMaxRecursionDepth)), metadata) + ctx = context.WithValue(ctx, "event_listen", event.Listen) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, data, DefaultMaxRecursionDepth)), metadata) metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } @@ -561,7 +573,7 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m } else if strings.Contains(m.Type, COLLECTION_TYPE) { m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION } - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, authData, DefaultMaxRecursionDepth)), m) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, authData, DefaultMaxRecursionDepth)), m) m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } @@ -591,7 +603,7 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C metadata.Type = originalType + " > header" metadata.Link = metadata.Link + "?tab=headers" metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, strings.Join(r.HeaderString, " "), DefaultMaxRecursionDepth)), metadata) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, strings.Join(r.HeaderString, " "), DefaultMaxRecursionDepth)), metadata) metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } @@ -600,7 +612,7 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C // Note: query parameters are handled separately u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/")) metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, u, DefaultMaxRecursionDepth)), metadata) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, u, DefaultMaxRecursionDepth)), metadata) metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } @@ -655,13 +667,13 @@ func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.C m.Type = originalType + " > raw" data := b.Raw m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_RAW - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data, DefaultMaxRecursionDepth)), m) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, data, DefaultMaxRecursionDepth)), m) m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN case "graphql": m.Type = originalType + " > graphql" data := b.GraphQL.Query + " " + b.GraphQL.Variables m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_GRAPHQL - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data, DefaultMaxRecursionDepth)), m) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, data, DefaultMaxRecursionDepth)), m) m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } } @@ -687,7 +699,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources. m.Type = originalType + " > response header" // TODO Note: for now, links to Postman responses do not include a more granular tab for the params/header/body, but when they do, we will need to update the metadata.Link info m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, strings.Join(response.HeaderString, " "), DefaultMaxRecursionDepth)), m) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, strings.Join(response.HeaderString, " "), DefaultMaxRecursionDepth)), m) m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } @@ -695,7 +707,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources. if response.Body != "" { m.Type = originalType + " > response body" m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY - s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, response.Body, DefaultMaxRecursionDepth)), m) + s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, response.Body, DefaultMaxRecursionDepth)), m) m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN } @@ -728,7 +740,7 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources. if valStr == "" { continue } - values = append(values, s.buildSubstituteSet(m, valStr, DefaultMaxRecursionDepth)...) + values = append(values, s.buildSubstituteSet(ctx, m, valStr, DefaultMaxRecursionDepth)...) } m.FieldType = m.Type + " variables" diff --git a/pkg/sources/postman/substitution.go b/pkg/sources/postman/substitution.go index 2b5b3ecbc..9f68f4f6e 100644 --- a/pkg/sources/postman/substitution.go +++ b/pkg/sources/postman/substitution.go @@ -4,6 +4,8 @@ import ( "fmt" "regexp" "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) var subRe = regexp.MustCompile(`\{\{[^{}]+\}\}`) @@ -52,12 +54,17 @@ func (s *Source) formatAndInjectKeywords(data []string) string { // buildSubstituteSet creates a set of substitutions for the given data // maxRecursionDepth is the maximum recursion depth to use for variable substitution -func (s *Source) buildSubstituteSet(metadata Metadata, data string, maxRecursionDepth int) []string { +func (s *Source) buildSubstituteSet( + ctx context.Context, + metadata Metadata, + data string, + maxRecursionDepth int, +) []string { var ret []string combos := make(map[string]struct{}) // Call buildSubstitution with initial depth of 0 and the maxRecursionDepth - s.buildSubstitution(data, metadata, &combos, 0, maxRecursionDepth) + s.buildSubstitution(ctx, data, metadata, combos, 0, maxRecursionDepth) for combo := range combos { ret = append(ret, combo) @@ -73,22 +80,26 @@ func (s *Source) buildSubstituteSet(metadata Metadata, data string, maxRecursion // depth is the current recursion depth // maxRecursionDepth is the maximum recursion depth to use for variable substitution func (s *Source) buildSubstitution( + ctx context.Context, data string, metadata Metadata, - combos *map[string]struct{}, + combos map[string]struct{}, depth int, maxRecursionDepth int, ) { // Limit recursion depth to prevent stack overflow if depth > maxRecursionDepth { - (*combos)[data] = struct{}{} + ctx.Logger().V(2).Info("Limited recursion depth", + "depth", depth, + ) + combos[data] = struct{}{} return } matches := removeDuplicateStr(subRe.FindAllString(data, -1)) if len(matches) == 0 { // No more substitutions to make, add to combos - (*combos)[data] = struct{}{} + combos[data] = struct{}{} return } @@ -117,14 +128,14 @@ func (s *Source) buildSubstitution( // Only mark substitution as made if we actually changed something if d != data { substitutionMade = true - s.buildSubstitution(d, metadata, combos, depth+1, maxRecursionDepth) + s.buildSubstitution(ctx, d, metadata, combos, depth+1, maxRecursionDepth) } } } // If no substitutions were made, add the current data if !substitutionMade { - (*combos)[data] = struct{}{} + combos[data] = struct{}{} } } diff --git a/pkg/sources/postman/substitution_test.go b/pkg/sources/postman/substitution_test.go index 5204ccb1a..e5ba5e44c 100644 --- a/pkg/sources/postman/substitution_test.go +++ b/pkg/sources/postman/substitution_test.go @@ -5,6 +5,8 @@ import ( "sort" "strings" "testing" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) func TestNewSubstitution(t *testing.T) { @@ -63,6 +65,8 @@ func TestSource_KeywordCombinations(t *testing.T) { } func TestSource_BuildSubstituteSet(t *testing.T) { + ctx := context.Background() + s := &Source{ sub: NewSubstitution(), } @@ -89,7 +93,7 @@ func TestSource_BuildSubstituteSet(t *testing.T) { } for _, tc := range testCases { - result := s.buildSubstituteSet(metadata, tc.data, DefaultMaxRecursionDepth) + result := s.buildSubstituteSet(ctx, metadata, tc.data, DefaultMaxRecursionDepth) if !reflect.DeepEqual(result, tc.expected) { t.Errorf("Expected substitution set: %v, got: %v", tc.expected, result) } @@ -158,6 +162,8 @@ func TestSource_FormatAndInjectKeywords(t *testing.T) { } func TestSource_BuildSubstitution_RecursionLimit(t *testing.T) { + ctx := context.Background() + s := &Source{ sub: NewSubstitution(), } @@ -229,9 +235,9 @@ func TestSource_BuildSubstitution_RecursionLimit(t *testing.T) { // Use custom maxDepth if provided, otherwise use default if tc.maxDepth > 0 { - s.buildSubstitution(tc.data, metadata, &combos, 0, tc.maxDepth) + s.buildSubstitution(ctx, tc.data, metadata, combos, 0, tc.maxDepth) } else { - s.buildSubstitution(tc.data, metadata, &combos, 0, DefaultMaxRecursionDepth) + s.buildSubstitution(ctx, tc.data, metadata, combos, 0, DefaultMaxRecursionDepth) } var result []string