Add allowlist feature to suppress test secrets

examples/whitelisted.yml

@@ -0,0 +1,20 @@
+- description: "RSA test keys"
+  values:
+    - |
+      -----BEGIN RSA PRIVATE KEY-----
+      MIIEpAIBAAKCAQEA...
+      -----END RSA PRIVATE KEY-----
+- description: "Demo API keys for documentation"
+  values:
+    - "demo_.*"
+    - "example-api-key-.*"
+    - "sk_test_.*"
+- description: "Decommissioned secrets (rotated out)"
+  values:
+    - "old-prod-key-2023"
+    - "^legacy-.*-deprecated$"
+- description: "Public certificates and well-known test tokens"
+  values:
+    - "^-----BEGIN CERTIFICATE-----"
+    - "password123"
+    - "token_1234567890abcdef"

main.go

@@ -65,6 +65,7 @@ var (
 	allowVerificationOverlap   = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
 	filterUnverified           = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
 	filterEntropy              = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
+	whitelistSecretsFile       = cli.Flag("whitelist-secrets", "Path to file with newline separated list of secrets to whitelist.").String()
 	scanEntireChunk            = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
 	compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
 	configFilename             = cli.Flag("config", "Path to configuration file.").ExistingFile()
@@ -518,6 +519,16 @@ func run(state overseer.State) {
 
 	verificationCacheMetrics := verificationcache.InMemoryMetrics{}
 
+	// Load whitelisted secrets if specified
+	var whitelistedSecrets map[string]struct{}
+	if *whitelistSecretsFile != "" {
+		whitelistedSecrets, err := detectors.LoadWhitelistedSecrets(*whitelistSecretsFile)
+		if err != nil {
+			logFatal(err, "failed to load whitelisted secrets")
+		}
+		logger.Info("loaded whitelisted secrets", "count", len(whitelistedSecrets), "file", *whitelistSecretsFile)
+	}
+
 	engConf := engine.Config{
 		Concurrency:       *concurrency,
 		ConfiguredSources: conf.Sources,
@@ -534,6 +545,7 @@ func run(state overseer.State) {
 		Dispatcher:               engine.NewPrinterDispatcher(printer),
 		FilterUnverified:         *filterUnverified,
 		FilterEntropy:            *filterEntropy,
+		WhitelistedSecrets:       whitelistedSecrets,
 		VerificationOverlap:      *allowVerificationOverlap,
 		Results:                  parsedResults,
 		PrintAvgDetectorTime:     *printAvgDetectorTime,

pkg/detectors/falsepositives.go

@@ -4,11 +4,14 @@ import (
 	_ "embed"
 	"fmt"
 	"math"
+	"os"
+	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
 
 	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	"gopkg.in/yaml.v3"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
@@ -29,6 +32,12 @@ type CustomFalsePositiveChecker interface {
 	IsFalsePositive(result Result) (bool, string)
 }
 
+// WhitelistEntry represents a whitelist entry in the YAML config
+type WhitelistEntry struct {
+	Description string   `yaml:"description,omitempty"` // Optional description for the whitelist
+	Values      []string `yaml:"values"`                // List of secret patterns/regexes to whitelist
+}
+
 var (
 	filter *ahocorasick.Trie
 
@@ -199,3 +208,100 @@ func FilterKnownFalsePositives(ctx context.Context, detector Detector, results [
 
 	return filteredResults
 }
+
+// FilterWhitelistedSecrets filters out results that match whitelisted secrets.
+// This allows users to specify known safe secrets that should not be reported.
+// Supports regex patterns.
+func FilterWhitelistedSecrets(ctx context.Context, results []Result, whitelistedSecrets map[string]struct{}) []Result {
+	if len(whitelistedSecrets) == 0 {
+		return results
+	}
+
+	var filteredResults []Result
+	for _, result := range results {
+		if len(result.Raw) == 0 {
+			filteredResults = append(filteredResults, result)
+			continue
+		}
+
+		isWhitelisted := false
+		var matchReason string
+
+		// Check if the raw secret matches any whitelisted secret
+		rawSecret := string(result.Raw)
+		if isWhitelisted, matchReason = isSecretWhitelisted(rawSecret, whitelistedSecrets); isWhitelisted {
+			ctx.Logger().V(4).Info("Skipping result: whitelisted secret", "result", maskSecret(rawSecret), "reason", matchReason)
+			continue
+		}
+
+		// Also check RawV2 if present
+		if result.RawV2 != nil {
+			rawV2Secret := string(result.RawV2)
+			if isWhitelisted, matchReason = isSecretWhitelisted(rawV2Secret, whitelistedSecrets); isWhitelisted {
+				ctx.Logger().V(4).Info("Skipping result: whitelisted secret", "result", maskSecret(rawV2Secret), "reason", matchReason)
+				continue
+			}
+		}
+
+		filteredResults = append(filteredResults, result)
+	}
+
+	return filteredResults
+}
+
+// LoadWhitelistedSecrets loads secrets from a YAML file that should be whitelisted.
+// The YAML format supports multiline secrets and includes optional descriptions.
+func LoadWhitelistedSecrets(yamlFile string) (map[string]struct{}, error) {
+	file, err := os.Open(yamlFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open whitelist file: %w", err)
+	}
+	defer file.Close()
+
+	var entries []WhitelistEntry
+	decoder := yaml.NewDecoder(file)
+	if err := decoder.Decode(&entries); err != nil {
+		return nil, fmt.Errorf("failed to parse YAML whitelist file: %w", err)
+	}
+
+	whitelistedSecrets := make(map[string]struct{})
+	for _, entry := range entries {
+		for _, value := range entry.Values {
+			if strings.TrimSpace(value) != "" { // Skip empty values
+				whitelistedSecrets[value] = struct{}{}
+			}
+		}
+	}
+
+	return whitelistedSecrets, nil
+}
+
+// isSecretWhitelisted checks if a secret matches any whitelisted pattern (exact string or regex)
+func isSecretWhitelisted(secret string, whitelistedSecrets map[string]struct{}) (bool, string) {
+	// First, try exact string matching for performance
+	if _, isWhitelisted := whitelistedSecrets[secret]; isWhitelisted {
+		return true, "exact match"
+	}
+
+	// Try regex matching
+	for pattern := range whitelistedSecrets {
+		if regex, err := regexp.Compile(pattern); err == nil {
+			if regex.MatchString(secret) {
+				return true, "regex match: " + pattern
+			}
+		}
+	}
+
+	return false, ""
+}
+
+// maskSecret masks a secret for safe logging by showing only the first and last few characters
+func maskSecret(secret string) string {
+	if len(secret) <= 8 {
+		return "***"
+	}
+	if len(secret) <= 16 {
+		return secret[:2] + "***" + secret[len(secret)-2:]
+	}
+	return secret[:4] + "***" + secret[len(secret)-4:]
+}