Add Circle CI source (#997)

* Add Circle CI source

* remove SHA1 line

* remove trim

README.md

@@ -96,7 +96,7 @@ trufflehog s3 --bucket=<bucket name> --only-verified
 TruffleHog v3 is a complete rewrite in Go with many new powerful features.
 
 - We've **added over 700 credential detectors that support active verification against their respective APIs**.
-- We've also added native **support for scanning GitHub, GitLab, filesystems, and S3**.
+- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, and Circle CI**.
 - **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
 
 
@@ -160,6 +160,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - S3
 - filesystem
 - syslog
+- circleci
 - file and stdin (coming soon)
 
 Each subcommand can have options that you can see with the `--help` flag provided to the sub command:

main.go

@@ -95,6 +95,9 @@ var (
 	syslogTLSCert  = syslogScan.Flag("cert", "Path to TLS cert.").String()
 	syslogTLSKey   = syslogScan.Flag("key", "Path to TLS key.").String()
 	syslogFormat   = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").String()
+
+	circleCiScan      = cli.Command("circleci", "Scan CircleCI")
+	circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
 )
 
 func init() {
@@ -293,6 +296,10 @@ func run(state overseer.State) {
 		if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan syslog.")
 		}
+	case circleCiScan.FullCommand():
+		if err = e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
+			logrus.WithError(err).Fatal("Failed to scan CircleCI.")
+		}
 	}
 	// asynchronously wait for scanning to finish and cleanup
 	go e.Finish(ctx)

pkg/engine/circleci.go

@@ -0,0 +1,48 @@
+package engine
+
+import (
+	"runtime"
+
+	"github.com/go-errors/errors"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
+)
+
+// ScanS3 scans CircleCI logs.
+func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
+	connection := &sourcespb.CircleCI{
+		Credential: &sourcespb.CircleCI_Token{
+			Token: token,
+		},
+	}
+
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		logrus.WithError(err).Error("failed to marshal Circle CI connection")
+		return err
+	}
+
+	circleSource := circleci.Source{}
+	err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
+	if err != nil {
+		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
+	}
+
+	e.sourcesWg.Add(1)
+	go func() {
+		defer common.RecoverWithExit(ctx)
+		defer e.sourcesWg.Done()
+		err := circleSource.Chunks(ctx, e.ChunksChan())
+		if err != nil {
+			logrus.WithError(err).Error("error scanning Circle CI")
+		}
+	}()
+	return nil
+}

pkg/engine/s3.go

@@ -42,7 +42,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
 	if err != nil {
-		logrus.WithError(err).Error("failed to marshal github connection")
+		logrus.WithError(err).Error("failed to marshal S3 connection")
 		return err
 	}
 
@@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
 		defer e.sourcesWg.Done()
 		err := s3Source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			logrus.WithError(err).Error("error scanning s3")
+			logrus.WithError(err).Error("error scanning S3")
 		}
 	}()
 	return nil

pkg/sources/circleci/circleci.go

@@ -0,0 +1,256 @@
+package circleci
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/go-errors/errors"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const (
+	baseURL = "https://circleci.com/api/v1.1/"
+)
+
+type Source struct {
+	name     string
+	sourceId int64
+	jobId    int64
+	verify   bool
+	sources.Progress
+	token  string
+	client *http.Client
+}
+
+// Ensure the Source satisfies the interface at compile time
+var _ sources.Source = (*Source)(nil)
+
+// Type returns the type of source.
+// It is used for matching source types in configuration and job input.
+func (s *Source) Type() sourcespb.SourceType {
+	return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
+}
+
+func (s *Source) SourceID() int64 {
+	return s.sourceId
+}
+
+func (s *Source) JobID() int64 {
+	return s.jobId
+}
+
+// Init returns an initialized Filesystem source.
+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
+	s.name = name
+	s.sourceId = sourceId
+	s.jobId = jobId
+	s.verify = verify
+	s.client = common.RetryableHttpClientTimeout(3)
+
+	var conn sourcespb.CircleCI
+	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
+	}
+
+	switch conn.Credential.(type) {
+	case *sourcespb.CircleCI_Token:
+		s.token = conn.GetToken()
+	}
+
+	return nil
+}
+
+// Chunks emits chunks of bytes over a channel.
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	projects, err := s.projects(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, proj := range projects {
+		builds, err := s.buildsForProject(ctx, proj)
+		if err != nil {
+			return err
+		}
+
+		for _, bld := range builds {
+			buildSteps, err := s.stepsForBuild(ctx, proj, bld)
+			if err != nil {
+				return err
+			}
+
+			for _, step := range buildSteps {
+				for _, action := range step.Actions {
+					err = s.chunkAction(ctx, proj, bld, action, step.Name, chunksChan)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+type project struct {
+	VCS      string `json:"vcs_type"`
+	Username string `json:"username"`
+	RepoName string `json:"reponame"`
+}
+
+func (s *Source) projects(ctx context.Context) ([]project, error) {
+	reqURL := fmt.Sprintf("%sprojects", baseURL)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Circle-Token", s.token)
+	req.Header.Set("Accept", "application/json")
+	res, err := s.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode > 399 && res.StatusCode < 500 {
+		return nil, fmt.Errorf("invalid credentials, status %d", res.StatusCode)
+	}
+
+	var projects []project
+	if err := json.NewDecoder(res.Body).Decode(&projects); err != nil {
+		return nil, err
+	}
+
+	return projects, nil
+}
+
+type build struct {
+	BuildNum int `json:"build_num"`
+}
+
+func (s *Source) buildsForProject(ctx context.Context, proj project) ([]build, error) {
+	reqURL := fmt.Sprintf("%sproject/%s/%s/%s", baseURL, proj.VCS, proj.Username, proj.RepoName)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Circle-Token", s.token)
+	req.Header.Set("Accept", "application/json")
+	res, err := s.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	var builds []build
+	if err := json.NewDecoder(res.Body).Decode(&builds); err != nil {
+		return nil, err
+	}
+
+	return builds, nil
+}
+
+type action struct {
+	Index     int    `json:"index"`
+	OutputURL string `json:"output_url"`
+}
+
+type buildStep struct {
+	Name    string   `json:"name"`
+	Actions []action `json:"actions"`
+}
+
+func (s *Source) stepsForBuild(ctx context.Context, proj project, bld build) ([]buildStep, error) {
+	reqURL := fmt.Sprintf("%sproject/%s/%s/%s/%d", baseURL, proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Circle-Token", s.token)
+	req.Header.Set("Accept", "application/json")
+	res, err := s.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	type buildRes struct {
+		Steps []buildStep `json:"steps"`
+	}
+
+	var bldRes buildRes
+	if err := json.NewDecoder(res.Body).Decode(&bldRes); err != nil {
+		return nil, err
+	}
+
+	return bldRes.Steps, nil
+}
+
+func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
+	req, err := http.NewRequest("GET", act.OutputURL, nil)
+	if err != nil {
+		return err
+	}
+	res, err := s.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	logOutput, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+
+	linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
+
+	chunk := &sources.Chunk{
+		SourceType: s.Type(),
+		SourceName: s.name,
+		SourceID:   s.SourceID(),
+		Data:       removeCircleSha1Line(logOutput),
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Circleci{
+				Circleci: &source_metadatapb.CircleCI{
+					VcsType:     proj.VCS,
+					Username:    proj.Username,
+					Repository:  proj.RepoName,
+					BuildNumber: int64(bld.BuildNum),
+					BuildStep:   stepName,
+					Link:        linkURL,
+				},
+			},
+		},
+		Verify: s.verify,
+	}
+
+	chunksChan <- chunk
+
+	return nil
+}
+
+func removeCircleSha1Line(input []byte) []byte {
+	// Split the input slice into a slice of lines
+	lines := bytes.Split(input, []byte("\n"))
+
+	// Iterate over the lines and add the ones that don't contain "CIRCLE_SHA1=" to the result slice
+	result := make([][]byte, 0, len(lines))
+	for _, line := range lines {
+		if !bytes.Contains(line, []byte("CIRCLE_SHA1=")) {
+			result = append(result, line)
+		}
+	}
+
+	// Join the lines in the result slice and return the resulting slice
+	return bytes.Join(result, []byte("\n"))
+}

pkg/sources/circleci/circleci_test.go

@@ -0,0 +1,93 @@
+package circleci
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestSource_Scan(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+
+	secret, err := common.GetTestSecret(ctx)
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+	}
+	token := secret.MustGetField("CIRCLECI_TOKEN")
+
+	type init struct {
+		name       string
+		verify     bool
+		connection *sourcespb.CircleCI
+	}
+	tests := []struct {
+		name               string
+		init               init
+		wantSourceMetadata *source_metadatapb.MetaData
+		wantErr            bool
+	}{
+		{
+			name: "get a chunk",
+			init: init{
+				name: "example repo",
+				connection: &sourcespb.CircleCI{
+					Credential: &sourcespb.CircleCI_Token{
+						Token: token,
+					},
+				},
+				verify: true,
+			},
+			wantSourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Circleci{
+					Circleci: &source_metadatapb.CircleCI{
+						VcsType:     "github",
+						Username:    "dustin-decker",
+						Repository:  "circle-ci",
+						BuildNumber: 2,
+						BuildStep:   "Spin up environment",
+						Link:        "https://app.circleci.com/pipelines/github/dustin-decker/circle-ci/2",
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Source{}
+
+			conn, err := anypb.New(tt.init.connection)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			chunksCh := make(chan *sources.Chunk, 1)
+			go func() {
+				err = s.Chunks(ctx, chunksCh)
+				if (err != nil) != tt.wantErr {
+					t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
+					return
+				}
+			}()
+			gotChunk := <-chunksCh
+			if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
+				t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}