diff --git a/pkg/detectors/azure_storage/storage.go b/pkg/detectors/azure_storage/storage.go new file mode 100644 index 000000000..8a56044ca --- /dev/null +++ b/pkg/detectors/azure_storage/storage.go @@ -0,0 +1,197 @@ +package azure_storage + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" + "time" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct { + client *http.Client + detectors.MultiPartCredentialProvider +} + +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses + + namePat = regexp.MustCompile(`(?i:Account[_.-]?Name|Storage[_.-]?(?:Account|Name))(?:.|\s){0,20}?\b([a-z0-9]{3,24})\b|([a-z0-9]{3,24})(?i:\.blob\.core\.windows\.net)`) // Names can only be lowercase alphanumeric. + // https://learn.microsoft.com/en-us/azure/storage/common/storage-use-emulator + testNames = map[string]struct{}{ + "devstoreaccount1": {}, + "storagesample": {}, + } + + keyPat = regexp.MustCompile(`(?i:(?:Access|Account|Storage)[_.-]?Key)(?:.|\s){0,25}?([a-zA-Z0-9+\/-]{86,88}={0,2})`) + testKeys = map[string]struct{}{ + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==": {}, + } +) + +func (s Scanner) Keywords() []string { + return []string{ + "DefaultEndpointsProtocol=http", "EndpointSuffix", "core.windows.net", + "AccountName", "Account_Name", "Account.Name", "Account-Name", + "StorageAccount", "Storage_Account", "Storage.Account", "Storage-Account", + "AccountKey", "Account_Key", "Account.Key", "Account-Key", + } +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_AzureStorage +} + +func (s Scanner) Description() string { + return "Azure Storage is a Microsoft-managed cloud service that provides storage that is highly available, secure, durable, scalable, and redundant. Azure Storage Account keys can be used to access and manage data within storage accounts." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + // Deduplicate results. + names := make(map[string]struct{}) + for _, matches := range namePat.FindAllStringSubmatch(dataStr, -1) { + var name string + if matches[1] != "" { + name = matches[1] + } else { + name = matches[2] + } + if _, ok := testNames[name]; ok { + continue + } + names[name] = struct{}{} + } + if len(names) == 0 { + return results, nil + } + + keys := make(map[string]struct{}) + for _, matches := range keyPat.FindAllStringSubmatch(dataStr, -1) { + key := matches[1] + if _, ok := testKeys[key]; ok { + continue + } + keys[key] = struct{}{} + } + if len(keys) == 0 { + return results, nil + } + + // Check results. + for name := range names { + var s1 detectors.Result + for key := range keys { + s1 = detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(key), + RawV2: []byte(fmt.Sprintf(`{"accountName":"%s","accountKey":"%s"}`, name, key)), + ExtraData: map[string]string{ + "Account_name": name, + }, + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + + isVerified, verificationErr := s.verifyMatch(ctx, client, name, key, s1.ExtraData) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, key) + } + + results = append(results, s1) + if s1.Verified { + break + } + } + } + + return results, nil +} + +type storageResponse struct { + Containers struct { + Container []container `xml:"Container"` + } `xml:"Containers"` +} + +type container struct { + Name string `xml:"Name"` +} + +func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, name string, key string, extraData map[string]string) (bool, error) { + // https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key + now := time.Now().UTC().Format(http.TimeFormat) + stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + name + "/\ncomp:list" + accountKeyBytes, _ := base64.StdEncoding.DecodeString(key) + h := hmac.New(sha256.New, accountKeyBytes) + h.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(h.Sum(nil)) + + url := "https://" + name + ".blob.core.windows.net/?comp=list" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + + req.Header.Set("x-ms-date", now) + req.Header.Set("x-ms-version", "2019-12-12") + req.Header.Set("Authorization", "SharedKey "+name+":"+signature) + + res, err := client.Do(req) + if err != nil { + // If the host is not found, we can assume that the accountName is not valid + if strings.Contains(err.Error(), "no such host") { + return false, nil + } + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + // parse response + response := storageResponse{} + if err := xml.NewDecoder(res.Body).Decode(&response); err != nil { + return false, err + } + + // update the extra data with container names only + if len(response.Containers.Container) > 0 { + var b strings.Builder + for i, c := range response.Containers.Container { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(c.Name) + } + extraData["container_names"] = b.String() + } + + return true, nil + case http.StatusForbidden: + // 403 if account id or key is invalid, or if the account is disabled + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} diff --git a/pkg/detectors/azurestorage/azurestorage_integration_test.go b/pkg/detectors/azure_storage/storage_integration_test.go similarity index 99% rename from pkg/detectors/azurestorage/azurestorage_integration_test.go rename to pkg/detectors/azure_storage/storage_integration_test.go index 1e45f122e..85c742b84 100644 --- a/pkg/detectors/azurestorage/azurestorage_integration_test.go +++ b/pkg/detectors/azure_storage/storage_integration_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package azurestorage +package azure_storage import ( "context" diff --git a/pkg/detectors/azure_storage/storage_test.go b/pkg/detectors/azure_storage/storage_test.go new file mode 100644 index 000000000..cf1689837 --- /dev/null +++ b/pkg/detectors/azure_storage/storage_test.go @@ -0,0 +1,328 @@ +package azure_storage + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestAzureStorage_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + // True Positive + // CONNECTION STRINGS + { + name: `connection_string_1`, + input: `DefaultEndpointsProtocol=https;AccountName=storagetest123;AccountKey=YutGV0Vlauqsobd5tPWz2AKwHhBXMEWsAH+rSbz0UZUfaMVj1CFrcNQK47ygmrC4vHmc7eOp1LdM+AStk5mMyA==;EndpointSuffix=core.windows.net`, + want: []string{`{"accountName":"storagetest123","accountKey":"YutGV0Vlauqsobd5tPWz2AKwHhBXMEWsAH+rSbz0UZUfaMVj1CFrcNQK47ygmrC4vHmc7eOp1LdM+AStk5mMyA=="}`}, + }, + { + name: `connection_string_2`, + input: `EndpointSuffix=core.windows.net;AccountKey=ldlKgoKPJhRjPJTkaC5c/QNqtu4sVQRc/teGJ0MZHbDYEHdvBV5z8JEfJK+evE87D7U8TzMZ0G2C+ASt2B4ifg==;AccountName=storagetest123;DefaultEndpointsProtocol=http`, + want: []string{`{"accountName":"storagetest123","accountKey":"ldlKgoKPJhRjPJTkaC5c/QNqtu4sVQRc/teGJ0MZHbDYEHdvBV5z8JEfJK+evE87D7U8TzMZ0G2C+ASt2B4ifg=="}`}, + }, + { + name: `connection_string_3`, + input: ` public const string SharedStorageKey = "DefaultEndpointsProtocol=https;AccountName=huntappstorage;AccountKey=rrttFty/b52ET/e8VqpMSN+ZqAUP7hcXVkdekrPX58gsMZyOCrE+igN07t3lyi7tAV0+OrJFBaDtMe06YJ2tFw==;EndpointSuffix=core.windows.net";`, + want: []string{`{"accountName":"huntappstorage","accountKey":"rrttFty/b52ET/e8VqpMSN+ZqAUP7hcXVkdekrPX58gsMZyOCrE+igN07t3lyi7tAV0+OrJFBaDtMe06YJ2tFw=="}`}, + }, + { + name: `connection_string_multiline`, + input: ` +export const DevelopmentConnectionString = 'DefaultEndpointsProtocol=http;AccountName=macdemostorage; + AccountKey=Jby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==; + QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;';`, + want: []string{`{"accountName":"macdemostorage","accountKey":"Jby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="}`}, + }, + + // LANGUAGES + // TODO: + // - https://github.com/Satyamk21/az204/blob/75f340c5bbfb34c1477a6885e216d5ae0972a380/Lab%203.txt#L22 + + // https://github.com/facebookincubator/velox/blob/98e958c0df498efd7cf44a2078cc71caeb7aed23/velox/connectors/hive/storage_adapters/abfs/tests/AzuriteServer.h#L32-L36 + { + name: `cpp`, + input: `static const std::string AzuriteAccountName{"storagetest123"}; +static const std::string AzuriteContainerName{"test"}; +// the default key of Azurite Server used for connection +static const std::string AzuriteAccountKey{ + "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="};`, + want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`}, + }, + // https://github.com/MicrosoftDX/Dash/blob/03c4bb55f9e84fd03ee943559c128c4d5c2a31c2/DashServer.Tests/RequestAuthTests.cs#L29 + { + name: `dotnet1`, + input: ` _ctx = InitializeConfigAndCreateTestBlobs(ctx, "datax1", new Dictionary + { + { "AccountName", "dashstorage1" }, + { "AccountKey", "8jqRVtXUWiEthgIhR+dFwrB8gh3lFuquvJQ1v4eabObIj7okI1cZIuzY8zZHmEdpcC0f+XlUkbFwAhjTfyrLIg==" }, + { "SecondaryAccountKey", "Klari9ZbVdFQ35aULCfqqehCsd136amhusMHWynTpz2Pg+GyQMJw3GH177hvEQbaZ2oeRYk3jw0mIaV3ehNIRg==" }, + },`, + want: []string{ + `{"accountName":"dashstorage1","accountKey":"8jqRVtXUWiEthgIhR+dFwrB8gh3lFuquvJQ1v4eabObIj7okI1cZIuzY8zZHmEdpcC0f+XlUkbFwAhjTfyrLIg=="}`, + `{"accountName":"dashstorage1","accountKey":"Klari9ZbVdFQ35aULCfqqehCsd136amhusMHWynTpz2Pg+GyQMJw3GH177hvEQbaZ2oeRYk3jw0mIaV3ehNIRg=="}`, + }, + }, + // https://github.com/Satyamk21/az204/blob/75f340c5bbfb34c1477a6885e216d5ae0972a380/Lab%203.txt#L11 + { + name: `dotnet2`, + input: `public class Program +{ + private const string blobServiceEndpoint = "https://k21storagemedia.blob.core.windows.net/"; + + private const string storageAccountName = "k21storagemedia"; + + private const string storageAccountKey = "DFdukxfl0SwO4NB91bi/FTPh9BMEKr6Z5wWf+tGDfXMakXvGVp/NDzAUjWc/9171OqoDvXSj1o8N+AStUk1GXg=="; + + + //The following code to create a new asynchronous Main method + public static async Task Main(string[] args)`, + want: []string{`{"accountName":"k21storagemedia","accountKey":"DFdukxfl0SwO4NB91bi/FTPh9BMEKr6Z5wWf+tGDfXMakXvGVp/NDzAUjWc/9171OqoDvXSj1o8N+AStUk1GXg=="}`}, + }, + // https://github.com/apache/camel/blob/main/test-infra/camel-test-infra-azure-common/src/test/java/org/apache/camel/test/infra/azure/common/services/AzuriteContainer.java#L25-L27 + { + name: `java`, + input: `public class AzuriteContainer extends GenericContainer { + public static final String DEFAULT_ACCOUNT_NAME = "storagetest123"; + public static final String DEFAULT_ACCOUNT_KEY + = "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="; + + public static final String IMAGE_NAME = "mcr.microsoft.com/azure-storage/azurite:3.27.0";`, + want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`}, + }, + // https://github.com/Azure/azure-storage-node/blob/6873387fc65bad6d577babe278be2ee2e6071493/test/common/connectionstringparsertests.js + { + name: `javascript`, + input: ` var parsedConnectionString = ServiceSettings.parseAndValidateKeys(defaultConnectionString + endpointsConnectionString, validKeys); + assert.equal(parsedConnectionString['DefaultEndpointsProtocol'], 'https'); + assert.equal(parsedConnectionString['AccountName'], 'storagetest123'); + assert.equal(parsedConnectionString['AccountKey'], 'KWPLd0rpW2T0U7K2pVpF8rYr1BgYtR7wYQk33AYiXeUoquiaY6o0TWqduxmPHlqeCNZ3LU0DHptbeIHy5l/Yhg=='); + assert.equal(parsedConnectionString['BlobEndpoint'], 'myBlobEndpoint'); + assert.equal(parsedConnectionString['QueueEndpoint'], 'myQueueEndpoint'); + assert.equal(parsedConnectionString['TableEndpoint'], 'myTableEndpoint');`, + want: []string{`{"accountName":"storagetest123","accountKey":"KWPLd0rpW2T0U7K2pVpF8rYr1BgYtR7wYQk33AYiXeUoquiaY6o0TWqduxmPHlqeCNZ3LU0DHptbeIHy5l/Yhg=="}`}, + }, + // https://github.com/nextcloud/server/blob/81a9e19ace190ea0a64d52d95d341e25c7ad618b/tests/preseed-config.php#L89 + { + name: `php`, + input: ` 'arguments' => [ + 'container' => 'test', + 'account_name' => 'storagetest123', + 'account_key' => 'qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==', + 'endpoint' => 'http://' . (getenv('DRONE') === 'true' ? 'azurite' : 'localhost') . ':10000/devstoreaccount1', + 'autocreate' => true + ]`, + want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`}, + }, + // https://github.com/Azure/azure-sdk-for-js/blob/2719dcfbe835a2da3003876dcb5d77efba95f912/sdk/cosmosdb/cosmos/test/public/common/_fakeTestSecrets.ts + { + name: `typescript`, + input: `export const name = + process.env.ACCOUNT_NAME || "storagename123"; +export const key = + process.env.ACCOUNT_KEY || + "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==";`, + want: []string{`{"accountName":"storagename123","accountKey":"C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="}`}, + }, + + // FORMATS + // TODO: Doesn't work. + // https://github.com/Azure/azure-quickstart-templates/blob/03e792429fbc65c9353335611933746364590b22/quickstarts/microsoft.datafactory/data-factory-hive-transformation/azuredeploy.parameters.json#L9C38-L9C38 + // { + // name: `json`, + // input: ` "storageAccountName": { + // "value": "changemeazurestorage" + //}, + //"storageAccountKey": { + // "value": "YA1gKAMY34PeVgEWPF8FdbQO+U0nFkd3SaFE4d32K16AYL/DowrTYun8anOdAiCnMkCiRYm+PxUh5mw7a7lVcA==" + //},`, + // want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`}, + // }, + // https://github.com/ClickHouse/ClickHouse/blob/eba52b318d67d85330c9c1781499b7ff27fb7c0e/tests/integration/test_storage_azure_blob_storage/configs/named_collections.xml + { + name: `xml`, + input: ` + storagetest123 + qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA== + `, + want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`}, + }, + // https://github.com/hubblestack/hubble/blob/f9b7bf38752bd16b27d050a3b8787652a1c6319b/hubblestack/fileserver/azurefs.py + { + name: `yaml1`, + input: ` azurefs: + - account_name: mystorage + account_key: 'fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA==' + container_name: my_container + proxy: 10.10.10.10:8080`, + want: []string{`{"accountName":"mystorage","accountKey":"fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA=="}`}, + }, + { + name: `yaml2`, + input: ` - name: filesharevolume + azureFile: + sharename: containershare + storageAccountName: newstore100033323 + storageAccountKey: Ar4/2iY8L0rEMeQaijINnfaMJr7vqjfbPgmJayw6Pu5l9ZI+GrFDm1uIWOqXk5RQLrTiXfBwWY6hAbPEIQqy1g==`, + want: []string{`{"accountName":"newstore100033323","accountKey":"Ar4/2iY8L0rEMeQaijINnfaMJr7vqjfbPgmJayw6Pu5l9ZI+GrFDm1uIWOqXk5RQLrTiXfBwWY6hAbPEIQqy1g=="}`}, + }, + // This was manually base64-decoded since that doesn't work in unit tests. + // https://github.com/fabric8io/configmapcontroller/blob/master/vendor/k8s.io/kubernetes/examples/azure_file/secret/azure-secret.yaml + { + name: `yaml_3`, + input: `apiVersion: v1 + kind: Secret + metadata: + name: azure-secret + type: Opaque + data: + azurestorageaccountname: k8stest + azurestorageaccountkey: xIF1zJbnnojFLMSkBp50mx02rHsMK2sjU7mFt4L13hoB7drAaJ8jD6+A443jJogV7y2FUOhQCWPmM6YaNHy7qg== +`, + want: []string{`{"accountName":"k8stest","accountKey":"xIF1zJbnnojFLMSkBp50mx02rHsMK2sjU7mFt4L13hoB7drAaJ8jD6+A443jJogV7y2FUOhQCWPmM6YaNHy7qg=="}`}, + }, + + // MISC + // https://github.com/Azure-Samples/nested-virtualization-image-builder/blob/cf0373a421343b00ce3d261be99ddced80deb55b/README.md?plain=1#L54 + { + name: `blob_url`, + input: `"name": "storagetest123.blob.core.windows.net", "accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="`, + want: []string{`{"accountName":"storagetest123","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`}, + }, + { + name: `random_cli_1`, + input: `go run .\main.go -debug -dest="https://kenfau.blob.core.windows.net/ss3/" -AzureDefaultAccountName="kenfoo" -AzureDefaultAccountKey="hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="`, + want: []string{ + `{"accountName":"kenfau","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`, + `{"accountName":"kenfoo","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`, + }, + }, + // - https://github.com/nwoolls/AzureStorageCleanup/blob/980e5cb163c78e9446e70d2513ba5a7ed9051a7a/README.md?plain=1#L24 + { + name: `random_cli_2`, + input: `AzureStorageCleanup.exe + -storagename storageaccount + -storagekey hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w== + -container sqlbackup + -mindaysold 60 + -searchpattern .* + -recursive + -whatif`, + want: []string{`{"accountName":"storageaccount","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`}, + }, + // https://github.com/dahlej/rpi-spark-titantic/blob/d00b8f5b4696aeb2113e9452c24bb31b7f9a0242/tmp.txt#L9 + { + name: `random_cli_3`, + input: `$ bin/spark-submit --master \ + k8s://test-cluster.eastus2.cloudapp.azure.com:443 \ + --deploy-mode cluster \ + --name copyLocations \ + --class io.timpark.CopyData \ + --conf spark.copydata.containerpath=wasb://containers@storagetest123.blob.core.windows.net \ + --conf spark.copydata.storageaccount=storagetest123 \ + --conf spark.copydata.storageaccountkey=hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w== \`, + want: []string{`{"accountName":"storagetest123","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`}, + }, + { + name: `custom_config_1`, + input: `driver := ArtifactDriver{ + AccountName: "storagetest123", + AccountKey: "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==", + Container: "test", + }`, + want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`}, + }, + // https://github.com/MicrosoftDX/Dash/blob/master/LoadTestDotNet/GetBlobCoded.cs + { + name: `storage_account_1`, + input: ` this.Context.Add("StorageEndPoint", "http://dashstorage3.blob.core.windows.net"); + this.Context.Add("StorageAccount", "dashstorage3"); + this.Context.Add("AccountKey", "TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="); + this.Context.Add("SendChunked", false);`, + want: []string{`{"accountName":"dashstorage3","accountKey":"TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="}`}, + }, + // https://github.com/kubecost/poc-common-configurations/blob/d626a48824a104e3089fc66ef57029f1e2212f6a/keys.txt#L18 + { + name: `storage_account_2`, + input: `AZ_cloud_integration_subscriptionId:0bd50fdf-c923-4e1e-850c-196ddSAMPLE +AZ_cloud_integration_azureStorageAccount:kubecostexport +AZ_cloud_integration_azureStorageAccessKey:TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ== +AZ_cloud_integration_azureStorageContainer:costexports`, + want: []string{`{"accountName":"kubecostexport","accountKey":"TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="}`}, + }, + + // False positives + { + name: `test_key`, + input: ` azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;`, + }, + { + name: `test_key_multiline`, + input: ` +export const DevelopmentConnectionString = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1; + AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==; + QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;';`, + }, + { + name: `invalid_key_1`, + input: ` docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net"`, + }, + { + name: `invalid_key_2`, + input: `PS C:\> Add-AzIotHubRoutingEndpoint -ResourceGroupName "myresourcegroup" -Name "myiothub" -EndpointName S1 -EndpointType AzureStorageContainer -EndpointResourceGroup resourcegroup1 -EndpointSubscriptionId 91d12343-a3de-345d-b2ea-135792468abc -ConnectionString 'DefaultEndpointsProtocol=https;AccountName=mystorage1;AccountKey=*****;EndpointSuffix=core.windows.net' -ContainerName container -Encoding json`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + if diff := cmp.Diff(test.want, results); diff != "" { + t.Errorf("%s expected %d results, received %d: (-want +got)\n%s", test.name, len(test.want), len(results), diff) + } + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/azurestorage/azurestorage.go b/pkg/detectors/azurestorage/azurestorage.go deleted file mode 100644 index 62797e361..000000000 --- a/pkg/detectors/azurestorage/azurestorage.go +++ /dev/null @@ -1,146 +0,0 @@ -package azurestorage - -import ( - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/xml" - "fmt" - "io" - "net/http" - "strings" - "time" - - regexp "github.com/wasilibs/go-re2" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct { - client *http.Client -} - -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - keyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P[^;]+);AccountKey=(?P[^;]+);EndpointSuffix=core\.windows\.net`) -) - -type storageResponse struct { - Containers struct { - Container []container `xml:"Container"` - } `xml:"Containers"` -} - -type container struct { - Name string `xml:"Name"` -} - -func (s Scanner) Keywords() []string { - return []string{"DefaultEndpointsProtocol=https;AccountName="} -} - -func (s Scanner) getClient() *http.Client { - if s.client != nil { - return s.client - } - return defaultClient -} - -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 3 { - continue - } - accountName := strings.TrimSpace(match[1]) - accountKey := strings.TrimSpace(match[2]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_AzureStorage, - Raw: []byte(accountKey), - ExtraData: map[string]string{ - "account_name": accountName, - }, - } - - if verify { - client := s.getClient() - - isVerified, verificationErr := verifyAzureStorageKey(ctx, client, accountName, accountKey, s1.ExtraData) - s1.Verified = isVerified - s1.SetVerificationError(verificationErr, accountKey) - } - - results = append(results, s1) - } - - return results, nil -} - -func verifyAzureStorageKey(ctx context.Context, client *http.Client, accountName, accountKey string, extraData map[string]string) (bool, error) { - now := time.Now().UTC().Format(http.TimeFormat) - stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + accountName + "/\ncomp:list" - accountKeyBytes, _ := base64.StdEncoding.DecodeString(accountKey) - h := hmac.New(sha256.New, accountKeyBytes) - h.Write([]byte(stringToSign)) - signature := base64.StdEncoding.EncodeToString(h.Sum(nil)) - - url := "https://" + accountName + ".blob.core.windows.net/?comp=list" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return false, err - } - req.Header.Set("x-ms-date", now) - req.Header.Set("x-ms-version", "2019-12-12") - req.Header.Set("Authorization", "SharedKey "+accountName+":"+signature) - - res, err := client.Do(req) - if err != nil { - // If the host is not found, we can assume that the accountName is not valid - if strings.Contains(err.Error(), "no such host") { - return false, nil - } - return false, err - } - defer res.Body.Close() - - switch res.StatusCode { - case http.StatusOK: - body, err := io.ReadAll(res.Body) - // ignore the error if failed because our keys has been verified. - if err == nil { - // parse response - response := storageResponse{} - if err := xml.Unmarshal(body, &response); err == nil { - // update the extra data with container names only - var containerNames []string - for _, c := range response.Containers.Container { - containerNames = append(containerNames, c.Name) - } - extraData["container_names"] = strings.Join(containerNames, ", ") - } - } - - return true, nil - case http.StatusForbidden: - // 403 if account id or key is invalid, or if the account is disabled - return false, nil - default: - return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - } -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_AzureStorage -} - -func (s Scanner) Description() string { - return "Azure Storage is a Microsoft-managed cloud service that provides storage that is highly available, secure, durable, scalable, and redundant. Azure Storage Account keys can be used to access and manage data within storage accounts." -} diff --git a/pkg/detectors/azurestorage/azurestorage_test.go b/pkg/detectors/azurestorage/azurestorage_test.go deleted file mode 100644 index 676900289..000000000 --- a/pkg/detectors/azurestorage/azurestorage_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package azurestorage - -import ( - "context" - "testing" - - "github.com/google/go-cmp/cmp" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" -) - -var ( - validPattern = ` - azure storage: - DefaultEndpointsProtocol=https;AccountName=myStorageAccount;AccountKey=myAccountKey12345==;EndpointSuffix=core.windows.net - ` - invalidPattern = ` - azure storage: - DefaultEndpointsProtocol=https;AccountName=;AccountKey=myAccountKey12345==;EndpointSuffix=core.ubuntu.net - ` -) - -func TestAzureStorage_Pattern(t *testing.T) { - d := Scanner{} - ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - - tests := []struct { - name string - input string - want []string - }{ - { - name: "valid pattern", - input: validPattern, - want: []string{"myAccountKey12345=="}, - }, - { - name: "invalid pattern", - input: invalidPattern, - want: nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { - t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) - return - } - - results, err := d.FromData(context.Background(), false, []byte(test.input)) - if err != nil { - t.Errorf("error = %v", err) - return - } - - if len(results) != len(test.want) { - if len(results) == 0 { - t.Errorf("did not receive result") - } else { - t.Errorf("expected %d results, only received %d", len(test.want), len(results)) - } - return - } - - actual := make(map[string]struct{}, len(results)) - for _, r := range results { - if len(r.RawV2) > 0 { - actual[string(r.RawV2)] = struct{}{} - } else { - actual[string(r.Raw)] = struct{}{} - } - } - expected := make(map[string]struct{}, len(test.want)) - for _, v := range test.want { - expected[v] = struct{}{} - } - - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) - } - }) - } -} diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 888fbb2d9..ff4a83b13 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -73,11 +73,11 @@ import ( azure_serviceprincipal_v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v1" azure_serviceprincipal_v2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_openai" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_storage" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchadminkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchquerykey" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurestorage" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baremetrics" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beamer" @@ -899,7 +899,7 @@ func buildDetectorList() []detectors.Detector { &azure_openai.Scanner{}, &azuresearchadminkey.Scanner{}, &azuresearchquerykey.Scanner{}, - &azurestorage.Scanner{}, + &azure_storage.Scanner{}, &bannerbear.Scanner{}, &baremetrics.Scanner{}, &beamer.Scanner{},