Add allowlist feature to suppress test secrets

examples/allowlist.yml

@@ -0,0 +1,20 @@
+- description: "RSA test keys"
+  values:
+    - |
+      -----BEGIN RSA PRIVATE KEY-----
+      MIIEpAIBAAKCAQEA...
+      -----END RSA PRIVATE KEY-----
+- description: "Demo API keys for documentation"
+  values:
+    - "demo_.*"
+    - "example-api-key-.*"
+    - "sk_test_.*"
+- description: "Decommissioned secrets (rotated out)"
+  values:
+    - "old-prod-key-2023"
+    - "^legacy-.*-deprecated$"
+- description: "Public certificates and well-known test tokens"
+  values:
+    - "^-----BEGIN CERTIFICATE-----"
+    - "password123"
+    - "token_1234567890abcdef"

main.go

@@ -65,6 +65,7 @@ var (
 	allowVerificationOverlap   = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
 	filterUnverified           = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
 	filterEntropy              = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
+	allowlistSecretsFile       = cli.Flag("allowlist-secrets", "Path to file with newline separated list of secrets to allowlist.").String()
 	scanEntireChunk            = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
 	compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
 	configFilename             = cli.Flag("config", "Path to configuration file.").ExistingFile()
@@ -518,6 +519,16 @@ func run(state overseer.State) {
 
 	verificationCacheMetrics := verificationcache.InMemoryMetrics{}
 
+	// Load allowlisted secrets if specified
+	var allowlistedSecrets map[string]struct{}
+	if *allowlistSecretsFile != "" {
+		allowlistedSecrets, err := detectors.LoadAllowlistedSecrets(*allowlistSecretsFile)
+		if err != nil {
+			logFatal(err, "failed to load allowlisted secrets")
+		}
+		logger.Info("loaded allowlisted secrets", "count", len(allowlistedSecrets), "file", *allowlistSecretsFile)
+	}
+
 	engConf := engine.Config{
 		Concurrency:       *concurrency,
 		ConfiguredSources: conf.Sources,
@@ -534,6 +545,7 @@ func run(state overseer.State) {
 		Dispatcher:               engine.NewPrinterDispatcher(printer),
 		FilterUnverified:         *filterUnverified,
 		FilterEntropy:            *filterEntropy,
+		AllowlistedSecrets:       allowlistedSecrets,
 		VerificationOverlap:      *allowVerificationOverlap,
 		Results:                  parsedResults,
 		PrintAvgDetectorTime:     *printAvgDetectorTime,

pkg/detectors/falsepositives.go

@@ -4,11 +4,14 @@ import (
 	_ "embed"
 	"fmt"
 	"math"
+	"os"
+	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
 
 	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	"gopkg.in/yaml.v3"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
@@ -29,6 +32,12 @@ type CustomFalsePositiveChecker interface {
 	IsFalsePositive(result Result) (bool, string)
 }
 
+// AllowlistEntry represents an allowlist entry in the YAML config
+type AllowlistEntry struct {
+	Description string   `yaml:"description,omitempty"` // Optional description for the allowlist
+	Values      []string `yaml:"values"`                // List of secret patterns/regexes to allowlist
+}
+
 var (
 	filter *ahocorasick.Trie
 
@@ -199,3 +208,100 @@ func FilterKnownFalsePositives(ctx context.Context, detector Detector, results [
 
 	return filteredResults
 }
+
+// FilterAllowlistedSecrets filters out results that match allowlisted secrets.
+// This allows users to specify known safe secrets that should not be reported.
+// Supports regex patterns.
+func FilterAllowlistedSecrets(ctx context.Context, results []Result, allowlistedSecrets map[string]struct{}) []Result {
+	if len(allowlistedSecrets) == 0 {
+		return results
+	}
+
+	var filteredResults []Result
+	for _, result := range results {
+		if len(result.Raw) == 0 {
+			filteredResults = append(filteredResults, result)
+			continue
+		}
+
+		isAllowlisted := false
+		var matchReason string
+
+		// Check if the raw secret matches any allowlisted secret
+		rawSecret := string(result.Raw)
+		if isAllowlisted, matchReason = isSecretAllowlisted(rawSecret, allowlistedSecrets); isAllowlisted {
+			ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", maskSecret(rawSecret), "reason", matchReason)
+			continue
+		}
+
+		// Also check RawV2 if present
+		if result.RawV2 != nil {
+			rawV2Secret := string(result.RawV2)
+			if isAllowlisted, matchReason = isSecretAllowlisted(rawV2Secret, allowlistedSecrets); isAllowlisted {
+				ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", maskSecret(rawV2Secret), "reason", matchReason)
+				continue
+			}
+		}
+
+		filteredResults = append(filteredResults, result)
+	}
+
+	return filteredResults
+}
+
+// LoadAllowlistedSecrets loads secrets from a YAML file that should be allowlisted.
+// The YAML format supports multiline secrets and includes optional descriptions.
+func LoadAllowlistedSecrets(yamlFile string) (map[string]struct{}, error) {
+	file, err := os.Open(yamlFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open allowlist file: %w", err)
+	}
+	defer file.Close()
+
+	var entries []AllowlistEntry
+	decoder := yaml.NewDecoder(file)
+	if err := decoder.Decode(&entries); err != nil {
+		return nil, fmt.Errorf("failed to parse YAML allowlist file: %w", err)
+	}
+
+	allowlistedSecrets := make(map[string]struct{})
+	for _, entry := range entries {
+		for _, value := range entry.Values {
+			if strings.TrimSpace(value) != "" { // Skip empty values
+				allowlistedSecrets[value] = struct{}{}
+			}
+		}
+	}
+
+	return allowlistedSecrets, nil
+}
+
+// isSecretAllowlisted checks if a secret matches any allowlisted pattern (exact string or regex)
+func isSecretAllowlisted(secret string, allowlistedSecrets map[string]struct{}) (bool, string) {
+	// First, try exact string matching for performance
+	if _, isAllowlisted := allowlistedSecrets[secret]; isAllowlisted {
+		return true, "exact match"
+	}
+
+	// Try regex matching
+	for pattern := range allowlistedSecrets {
+		if regex, err := regexp.Compile(pattern); err == nil {
+			if regex.MatchString(secret) {
+				return true, "regex match: " + pattern
+			}
+		}
+	}
+
+	return false, ""
+}
+
+// maskSecret masks a secret for safe logging by showing only the first and last few characters
+func maskSecret(secret string) string {
+	if len(secret) <= 8 {
+		return "***"
+	}
+	if len(secret) <= 16 {
+		return secret[:2] + "***" + secret[len(secret)-2:]
+	}
+	return secret[:4] + "***" + secret[len(secret)-4:]
+}