Add allowlist feature to suppress test secrets

examples/allowlist.yml

@@ -0,0 +1,20 @@
+- description: "RSA test keys"
+  values:
+    - |
+      -----BEGIN RSA PRIVATE KEY-----
+      MIIEpAIBAAKCAQEA...
+      -----END RSA PRIVATE KEY-----
+- description: "Demo API keys for documentation"
+  values:
+    - "demo_.*"
+    - "example-api-key-.*"
+    - "sk_test_.*"
+- description: "Decommissioned secrets (rotated out)"
+  values:
+    - "old-prod-key-2023"
+    - "^legacy-.*-deprecated$"
+- description: "Public certificates and well-known test tokens"
+  values:
+    - "^-----BEGIN CERTIFICATE-----"
+    - "password123"
+    - "token_1234567890abcdef"

examples/generic.yml

@@ -13,3 +13,13 @@ detectors:
   regex:
     # borrowing the gitleaks generic-api-key regex
     generic-api-key: "(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\\-_\\t .]{0,20})(?:[\\s|']|[\\s|\"]){0,3}(?:=|>|:{1,3}=|\\|\\|:|<=|=>|:|\\?=)(?:'|\"|\\s|=|\\x60){0,5}([0-9a-z\\-_.=]{10,150})(?:['|\"|\\n|\\r|\\s|\\x60|;]|$)"
+allowlists:
+  - description: "Test allowlist"
+    values:
+      - "https://user:password@example.com"
+      - |
+        -----BEGIN OPENSSH PRIVATE KEY-----
+        b3BlbnNzaC1rZXktdjEAAAAACm
+  - description: "Ignore AWS access keys"
+    values:
+      - "AKIA*"

main.go

@@ -65,6 +65,7 @@ var (
 	allowVerificationOverlap   = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
 	filterUnverified           = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
 	filterEntropy              = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
+	allowlistSecretsFile       = cli.Flag("allowlist-secrets-file", "Path to YAML file with secrets to allowlist. See examples/allowlist.yml for format.").String()
 	scanEntireChunk            = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
 	compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
 	configFilename             = cli.Flag("config", "Path to configuration file.").ExistingFile()
@@ -520,6 +521,25 @@ func run(state overseer.State) {
 
 	verificationCacheMetrics := verificationcache.InMemoryMetrics{}
 
+	// Load allowlisted secrets if specified
+	var allowlistedSecrets []detectors.AllowlistEntry
+	if *allowlistSecretsFile != "" {
+		allowlistedSecrets, err = detectors.LoadAllowlistedSecrets(*allowlistSecretsFile)
+		if err != nil {
+			logFatal(err, "failed to load allowlisted secrets")
+		}
+	}
+	allowListedSecrets := append(allowlistedSecrets, conf.Allowlists...)
+	compiledAllowlist := detectors.CompileAllowlistPatterns(allowListedSecrets)
+
+	logger.Info(
+		"loaded allowlisted secrets",
+		"exact_matches", len(compiledAllowlist.ExactMatches),
+		"regex_patterns", len(compiledAllowlist.CompiledRegexes),
+		"total", len(compiledAllowlist.ExactMatches)+len(compiledAllowlist.CompiledRegexes),
+		"file", *allowlistSecretsFile,
+	)
+
 	engConf := engine.Config{
 		Concurrency:       *concurrency,
 		ConfiguredSources: conf.Sources,
@@ -536,6 +556,7 @@ func run(state overseer.State) {
 		Dispatcher:               engine.NewPrinterDispatcher(printer),
 		FilterUnverified:         *filterUnverified,
 		FilterEntropy:            *filterEntropy,
+		AllowlistedSecrets:       compiledAllowlist,
 		VerificationOverlap:      *allowVerificationOverlap,
 		Results:                  parsedResults,
 		PrintAvgDetectorTime:     *printAvgDetectorTime,

pkg/config/config.go

@@ -23,8 +23,9 @@ import (
 
 // Config holds user supplied configuration.
 type Config struct {
-	Sources   []sources.ConfiguredSource
-	Detectors []detectors.Detector
+	Sources    []sources.ConfiguredSource
+	Detectors  []detectors.Detector
+	Allowlists []detectors.AllowlistEntry
 }
 
 // Read parses a given filename into a Config.
@@ -66,9 +67,19 @@ func NewYAML(input []byte) (*Config, error) {
 		sourceConfigs = append(sourceConfigs, src)
 	}
 
+	// Convert allowlist entries to Go structs.
+	var allowlistConfigs []detectors.AllowlistEntry
+	for _, pbAllowlist := range inputYAML.Allowlists {
+		allowlistConfigs = append(allowlistConfigs, detectors.AllowlistEntry{
+			Description: pbAllowlist.GetDescription(),
+			Values:      pbAllowlist.GetValues(),
+		})
+	}
+
 	return &Config{
-		Detectors: detectorConfigs,
-		Sources:   sourceConfigs,
+		Detectors:  detectorConfigs,
+		Sources:    sourceConfigs,
+		Allowlists: allowlistConfigs,
 	}, nil
 }
 

pkg/detectors/falsepositives.go

@@ -3,14 +3,20 @@ package detectors
 import (
 	_ "embed"
 	"fmt"
+	"io"
 	"math"
+	"os"
+	"regexp"
+	"slices"
 	"strings"
 	"unicode"
 	"unicode/utf8"
 
 	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	"gopkg.in/yaml.v3"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 )
 
 var (
@@ -29,6 +35,19 @@ type CustomFalsePositiveChecker interface {
 	IsFalsePositive(result Result) (bool, string)
 }
 
+// AllowlistEntry represents an allowlist entry in the YAML config
+type AllowlistEntry struct {
+	Description string   `yaml:"description,omitempty"` // Optional description for the allowlist
+	Values      []string `yaml:"values"`                // List of secret patterns/regexes to allowlist
+}
+
+// CompiledAllowlist holds both exact string matches and compiled regex patterns for efficient matching
+type CompiledAllowlist struct {
+	ExactMatches    map[string]struct{} // For exact string matching (O(1) lookup)
+	CompiledRegexes []*regexp.Regexp    // Pre-compiled regex patterns
+	RegexPatterns   []string            // Original regex patterns (for logging/debugging)
+}
+
 var (
 	filter *ahocorasick.Trie
 
@@ -199,3 +218,117 @@ func FilterKnownFalsePositives(ctx context.Context, detector Detector, results [
 
 	return filteredResults
 }
+
+// FilterAllowlistedSecrets filters out results that match allowlisted secrets.
+// This allows users to specify known safe secrets that should not be reported.
+// Supports regex patterns.
+func FilterAllowlistedSecrets(ctx context.Context, results []Result, allowlist *CompiledAllowlist) []Result {
+	if allowlist == nil || (len(allowlist.ExactMatches) == 0 && len(allowlist.CompiledRegexes) == 0) {
+		return results
+	}
+
+	return slices.DeleteFunc(results, func(result Result) bool {
+		if len(result.Raw) == 0 {
+			return false // Keep results with empty Raw
+		}
+
+		// Check if the raw secret matches any allowlisted secret
+		rawSecret := string(result.Raw)
+		log.RedactGlobally(rawSecret)
+		if isAllowlisted, matchReason := isSecretAllowlisted(rawSecret, allowlist); isAllowlisted {
+			ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", rawSecret, "reason", matchReason)
+			return true // Delete this result
+		}
+
+		// Also check RawV2 if present
+		if result.RawV2 != nil {
+			rawV2Secret := string(result.RawV2)
+			if isAllowlisted, matchReason := isSecretAllowlisted(rawV2Secret, allowlist); isAllowlisted {
+				ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", rawV2Secret, "reason", matchReason)
+				return true // Delete this result
+			}
+		}
+
+		return false // Keep this result
+	})
+}
+
+// LoadAllowlistedSecrets loads secrets from a YAML file that should be allowlisted.
+// The YAML format supports multiline secrets and includes optional descriptions.
+// Returns a CompiledAllowlist with pre-compiled regex patterns for efficient matching.
+func LoadAllowlistedSecrets(yamlFile string) ([]AllowlistEntry, error) {
+	file, err := os.Open(yamlFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open allowlist file: %w", err)
+	}
+	defer file.Close()
+
+	// Read the entire file content
+	content, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read allowlist file: %w", err)
+	}
+
+	var allowList []AllowlistEntry
+	if err := yaml.Unmarshal(content, &allowList); err != nil {
+		return nil, fmt.Errorf("failed to parse YAML allowlist file: %w", err)
+	}
+
+	return allowList, nil
+}
+
+// CompileAllowlistPatterns compiles a list of patterns into a CompiledAllowlist.
+// All patterns are first attempted to be compiled as regex. If compilation fails,
+// they are treated as exact string matches.
+func CompileAllowlistPatterns(allowList []AllowlistEntry) *CompiledAllowlist {
+	compiledAllowlist := &CompiledAllowlist{
+		ExactMatches:    make(map[string]struct{}, 0),
+		CompiledRegexes: make([]*regexp.Regexp, 0),
+		RegexPatterns:   make([]string, 0),
+	}
+
+	for _, entry := range allowList {
+		for _, pattern := range entry.Values {
+			pattern = strings.TrimSpace(pattern)
+			if pattern == "" {
+				continue // Skip empty patterns
+			}
+
+			// Always try to compile as regex first
+			if compiledRegex, err := regexp.Compile(pattern); err == nil {
+				// Successfully compiled as regex
+				compiledAllowlist.CompiledRegexes = append(compiledAllowlist.CompiledRegexes, compiledRegex)
+				compiledAllowlist.RegexPatterns = append(compiledAllowlist.RegexPatterns, pattern)
+			} else {
+				// Invalid regex, treat as exact string match
+				compiledAllowlist.ExactMatches[pattern] = struct{}{}
+			}
+		}
+	}
+
+	return compiledAllowlist
+}
+
+// isSecretAllowlisted checks if a secret matches any allowlisted pattern (exact string or regex)
+func isSecretAllowlisted(secret string, allowlist *CompiledAllowlist) (bool, string) {
+	if allowlist == nil {
+		return false, ""
+	}
+
+	// Trim all whitespace (spaces, tabs, newlines, carriage returns) from the secret
+	secret = strings.TrimSpace(secret)
+
+	// First, try exact string matching for performance (O(1) lookup)
+	if _, isAllowlisted := allowlist.ExactMatches[secret]; isAllowlisted {
+		return true, "exact match"
+	}
+
+	// Try pre-compiled regex patterns
+	for i, compiledRegex := range allowlist.CompiledRegexes {
+		if compiledRegex.MatchString(secret) {
+			return true, "regex match: " + allowlist.RegexPatterns[i]
+		}
+	}
+
+	return false, ""
+}