Postman source handle different JSON info for headers (#3995)

pkg/output/plain.go

@@ -117,9 +117,9 @@ func structToMap(obj any) (m map[string]map[string]any, err error) {
 	}
 	err = json.Unmarshal(data, &m)
 	// Due to PostmanLocationType protobuf field being an enum, we want to be able to assign the string value of the enum to the field without needing to create another Protobuf field.
-	// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID is filled since every secret in the Postman source
-	// should have a valid workspace ID and the 0 value is considered nil for integers.
-	if m["Postman"]["workspace_uuid"] != nil {
+	// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID or collection ID is filled since every secret
+	// in the Postman source should have a valid workspace ID or collection ID and the 0 value is considered nil for integers.
+	if m["Postman"]["workspace_uuid"] != nil || m["Postman"]["collection_id"] != nil {
 		if m["Postman"]["location_type"] == nil {
 			m["Postman"]["location_type"] = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN.String()
 		} else {

pkg/sources/postman/postman.go

@@ -422,7 +422,7 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,
 		metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
 	}
 
-	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, data)), metadata)
+	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, data)), metadata)
 	metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
@@ -520,7 +520,7 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
 	} else if strings.Contains(m.Type, COLLECTION_TYPE) {
 		m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
 	}
-	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, authData)), m)
+	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, authData)), m)
 	m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
@@ -529,9 +529,9 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
 	originalType := metadata.Type
 
 	// Add in var procesisng for headers
-	if r.Header != nil {
+	if r.HeaderKeyValue != nil {
 		vars := VariableData{
-			KeyValues: r.Header,
+			KeyValues: r.HeaderKeyValue,
 		}
 		metadata.Type = originalType + " > header"
 		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
@@ -539,12 +539,20 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
 		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
+	if r.HeaderString != nil {
+		metadata.Type = originalType + " > header"
+		metadata.Link = metadata.Link + "?tab=headers"
+		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, strings.Join(r.HeaderString, " "))), metadata)
+		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
+	}
+
 	if r.URL.Raw != "" {
 		metadata.Type = originalType + " > request URL (no query parameters)"
 		// Note: query parameters are handled separately
 		u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
 		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, u)), metadata)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, u)), metadata)
 		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
@@ -595,13 +603,13 @@ func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.C
 		m.Type = originalType + " > raw"
 		data := b.Raw
 		m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_RAW
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	case "graphql":
 		m.Type = originalType + " > graphql"
 		data := b.GraphQL.Query + " " + b.GraphQL.Variables
 		m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_GRAPHQL
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 }
@@ -613,9 +621,9 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
 	}
 	originalType := m.Type
 
-	if response.Header != nil {
+	if response.HeaderKeyValue != nil {
 		vars := VariableData{
-			KeyValues: response.Header,
+			KeyValues: response.HeaderKeyValue,
 		}
 		m.Type = originalType + " > response header"
 		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
@@ -623,11 +631,19 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
+	if response.HeaderString != nil {
+		m.Type = originalType + " > response header"
+		// TODO Note: for now, links to Postman responses do not include a more granular tab for the params/header/body, but when they do, we will need to update the metadata.Link info
+		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, strings.Join(response.HeaderString, " "))), m)
+		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
+	}
+
 	// Body in a response is just a string
 	if response.Body != "" {
 		m.Type = originalType + " > response body"
 		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, response.Body)), m)
+		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, response.Body)), m)
 		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
@@ -660,7 +676,7 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
 		if valStr == "" {
 			continue
 		}
-		values = append(values, s.buildSubstitueSet(m, valStr)...)
+		values = append(values, s.buildSubstituteSet(m, valStr)...)
 	}
 
 	m.FieldType = m.Type + " variables"

pkg/sources/postman/postman_client.go

@@ -44,12 +44,12 @@ type IDNameUUID struct {
 }
 
 type KeyValue struct {
-	Key          string      `json:"key"`
-	Value        interface{} `json:"value"`
-	Enabled      bool        `json:"enabled,omitempty"`
-	Type         string      `json:"type,omitempty"`
-	SessionValue string      `json:"sessionValue,omitempty"`
-	Id           string      `json:"id,omitempty"`
+	Key          string `json:"key"`
+	Value        any    `json:"value"`
+	Enabled      bool   `json:"enabled,omitempty"`
+	Type         string `json:"type,omitempty"`
+	SessionValue string `json:"sessionValue,omitempty"`
+	Id           string `json:"id,omitempty"`
 }
 
 type VariableData struct {
@@ -136,12 +136,14 @@ type Script struct {
 }
 
 type Request struct {
-	Auth        Auth       `json:"auth,omitempty"`
-	Method      string     `json:"method"`
-	Header      []KeyValue `json:"header,omitempty"`
-	Body        Body       `json:"body,omitempty"` //Need to update with additional options
-	URL         URL        `json:"url"`
-	Description string     `json:"description,omitempty"`
+	Auth           Auth            `json:"auth,omitempty"`
+	Method         string          `json:"method"`
+	HeaderRaw      json.RawMessage `json:"header,omitempty"`
+	HeaderKeyValue []KeyValue
+	HeaderString   []string
+	Body           Body   `json:"body,omitempty"` //Need to update with additional options
+	URL            URL    `json:"url"`
+	Description    string `json:"description,omitempty"`
 }
 
 type Body struct {
@@ -171,12 +173,14 @@ type URL struct {
 }
 
 type Response struct {
-	ID              string     `json:"id"`
-	Name            string     `json:"name,omitempty"`
-	OriginalRequest Request    `json:"originalRequest,omitempty"`
-	Header          []KeyValue `json:"header,omitempty"`
-	Body            string     `json:"body,omitempty"`
-	UID             string     `json:"uid,omitempty"`
+	ID              string          `json:"id"`
+	Name            string          `json:"name,omitempty"`
+	OriginalRequest Request         `json:"originalRequest,omitempty"`
+	HeaderRaw       json.RawMessage `json:"header,omitempty"`
+	HeaderKeyValue  []KeyValue
+	HeaderString    []string
+	Body            string `json:"body,omitempty"`
+	UID             string `json:"uid,omitempty"`
 }
 
 // A Client manages communication with the Postman API.
@@ -375,5 +379,30 @@ func (c *Client) GetCollection(ctx context.Context, collection_uuid string) (Col
 		return Collection{}, fmt.Errorf("could not unmarshal JSON for collection (%s): %w", collection_uuid, err)
 	}
 
+	// Loop used to deal with seeing whether a request/response header is a string or a key value pair
+	for i := range obj.Collection.Items {
+		if obj.Collection.Items[i].Request.HeaderRaw != nil {
+			if err := json.Unmarshal(obj.Collection.Items[i].Request.HeaderRaw, &obj.Collection.Items[i].Request.HeaderKeyValue); err == nil {
+			} else if err := json.Unmarshal(obj.Collection.Items[i].Request.HeaderRaw, &obj.Collection.Items[i].Request.HeaderString); err == nil {
+			} else {
+				return Collection{}, fmt.Errorf("could not unmarshal request header JSON for collection (%s): %w", collection_uuid, err)
+			}
+		}
+
+		for j := range obj.Collection.Items[i].Response {
+			if err := json.Unmarshal(obj.Collection.Items[i].Response[j].OriginalRequest.HeaderRaw, &obj.Collection.Items[i].Response[j].OriginalRequest.HeaderKeyValue); err == nil {
+			} else if err := json.Unmarshal(obj.Collection.Items[i].Response[j].OriginalRequest.HeaderRaw, &obj.Collection.Items[i].Response[j].OriginalRequest.HeaderString); err == nil {
+			} else {
+				return Collection{}, fmt.Errorf("could not unmarshal original request header in response JSON for collection (%s): %w", collection_uuid, err)
+			}
+
+			if err := json.Unmarshal(obj.Collection.Items[i].Response[j].HeaderRaw, &obj.Collection.Items[i].Response[j].HeaderKeyValue); err == nil {
+			} else if err := json.Unmarshal(obj.Collection.Items[i].Response[j].HeaderRaw, &obj.Collection.Items[i].Response[j].HeaderString); err == nil {
+			} else {
+				return Collection{}, fmt.Errorf("could not unmarshal response header JSON for collection (%s): %w", collection_uuid, err)
+			}
+		}
+	}
+
 	return obj.Collection, nil
 }

pkg/sources/postman/postman_test.go

@@ -333,3 +333,116 @@ func TestSource_ScanGeneralRateLimit(t *testing.T) {
 		t.Errorf("Rate limiting not working as expected. Elapsed time: %v seconds, expected at least %v seconds", elapsed.Seconds(), (float64(numRequests)-1)/5)
 	}
 }
+
+func TestSource_UnmarshalMultipleHeaderTypes(t *testing.T) {
+	defer gock.Off()
+	// Mock a collection with request and response headers of KeyValue type
+	gock.New("https://api.getpostman.com").
+		Get("/collections/1234-abc1").
+		Reply(200).
+		BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
+	 	"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},
+	 	"item":[{"name":"echo","id":"req-ues-t1","protocolProfileBehavior":{"disableBodyPruning":true},"request":{"method":"GET","header":[{"key":"Date","value":"Fri, 21 Mar 2025 17:38:58 GMT"}]},
+	 	"response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":[{"key":"Date","value":"Fri, 21 Mar 2025 17:38:58 GMT"}],
+	 	"url":{"raw":"postman-echo.com/get","host":["postman-echo","com"],"path":["get"]}},"status":"OK","code":200,"_postman_previewlanguage":"json",
+	 	"header":[{"key":"Date","value":"Fri, 21 Mar 2025 17:38:58 GMT"},{"key":"Content-Type","value":"application/json; charset=utf-8"},{"key":"Content-Length","value":"508"},
+	 	{"key":"Connection","value":"keep-alive"},{"key":"Server","value":"nginx"},{"key":"ETag","value":"random-string"},
+		{"key":"set-cookie","value":"sails.sid=long-string; Path=/; HttpOnly"}],"cookie":[], "responseTime":null,"body":"{response-body}","uid":"1234-res-pon-se1"}],"uid":"1234-req-ues-t1"}]}}`)
+	// Mock a collection with request and response headers of string type
+	gock.New("https://api.getpostman.com").
+		Get("/collections/1234-def1").
+		Reply(200).
+		BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
+	 	"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-def1"},
+	 	"item":[{"name":"echo","id":"req-ues-t1","protocolProfileBehavior":{"disableBodyPruning":true},"request":{"method":"GET","header":["request-header-string"]},
+	 	"response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":["request-header-string"],
+	 	"url":{"raw":"postman-echo.com/get","host":["postman-echo","com"],"path":["get"]}},"status":"OK","code":200,"_postman_previewlanguage":"json",
+	 	"header":["response-header-string"],"cookie":[], "responseTime":null,"body":"{response-body}","uid":"1234-res-pon-se1"}],"uid":"1234-req-ues-t1"}]}}`)
+
+	ctx := context.Background()
+	s, conn := createTestSource(&sourcespb.Postman{
+		Credential: &sourcespb.Postman_Token{
+			Token: "super-secret-token",
+		},
+	})
+	err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)
+	if err != nil {
+		t.Fatalf("init error: %v", err)
+	}
+	gock.InterceptClient(s.client.HTTPClient)
+	defer gock.RestoreClient(s.client.HTTPClient)
+
+	collectionIds := []string{"1234-abc1", "1234-def1"}
+	for _, collectionId := range collectionIds {
+		_, err := s.client.GetCollection(ctx, collectionId)
+		if err != nil {
+			t.Fatalf("failed to get collection: %v", err)
+		}
+	}
+}
+
+// The purpose of the TestSource_HeadersScanning test is to check that at least one of the fields HeaderKeyValue or HeaderString are non-null after unmarshalling and that chunks can
+// be generated from them.
+func TestSource_HeadersScanning(t *testing.T) {
+	defer gock.Off()
+	// Mock a collection with request and response headers of KeyValue type
+	gock.New("https://api.getpostman.com").
+		Get("/collections/1234-abc1").
+		Reply(200).
+		BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
+        "updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},
+        "item":[{"name":"echo","id":"req-ues-t1", "request":{"method":"GET","header":[{"key":"token","value":"keyword1"}]},
+        "response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":[{"key":"token","value":"keyword1"}]},
+        "header":[{"key":"token","value":"keyword1"}]}],"uid":"1234-req-ues-t1"}]}}`)
+	// Mock a collection with request and response headers of string type
+	gock.New("https://api.getpostman.com").
+		Get("/collections/1234-def1").
+		Reply(200).
+		BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
+        "updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-def1"},
+        "item":[{"name":"echo","id":"req-ues-t1","protocolProfileBehavior":{"disableBodyPruning":true},"request":{"method":"GET","header":["keyword1-request-header-string"]},
+        "response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":["keyword1-request-header-string"]},
+        "header":["keyword1-response-header-string"]}],"uid":"1234-req-ues-t1"}]}}`)
+
+	ctx := context.Background()
+	s, conn := createTestSource(&sourcespb.Postman{
+		Credential: &sourcespb.Postman_Token{
+			Token: "super-secret-token",
+		},
+	})
+
+	// Add detector keywords to trigger chunk generation
+	s.DetectorKeywords = map[string]struct{}{
+		"keyword1": {},
+	}
+	s.keywords = map[string]struct{}{
+		"keyword1": {},
+	}
+
+	err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)
+	if err != nil {
+		t.Fatalf("init error: %v", err)
+	}
+	gock.InterceptClient(s.client.HTTPClient)
+	defer gock.RestoreClient(s.client.HTTPClient)
+
+	chunksChan := make(chan *sources.Chunk, 10)
+	collectionIds := []string{"1234-abc1", "1234-def1"}
+
+	for _, collectionId := range collectionIds {
+		collection, err := s.client.GetCollection(ctx, collectionId)
+		if err != nil {
+			t.Fatalf("failed to get collection: %v", err)
+		}
+		s.scanCollection(ctx, chunksChan, Metadata{CollectionInfo: collection.Info}, collection)
+	}
+
+	close(chunksChan)
+	chunksReceived := len(chunksChan)
+
+	if chunksReceived == 0 {
+		t.Errorf("No chunks were generated from the mock data")
+	} else {
+		t.Logf("Generated %d chunks from the mock data", chunksReceived)
+	}
+}

pkg/sources/postman/substitution.go

@@ -47,7 +47,7 @@ func (s *Source) formatAndInjectKeywords(data []string) string {
 	return strings.Join(ret, "")
 }
 
-func (s *Source) buildSubstitueSet(metadata Metadata, data string) []string {
+func (s *Source) buildSubstituteSet(metadata Metadata, data string) []string {
 	var ret []string
 	combos := make(map[string]struct{})
 

pkg/sources/postman/substitution_test.go

@@ -89,7 +89,7 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		result := s.buildSubstitueSet(metadata, tc.data)
+		result := s.buildSubstituteSet(metadata, tc.data)
 		if !reflect.DeepEqual(result, tc.expected) {
 			t.Errorf("Expected substitution set: %v, got: %v", tc.expected, result)
 		}