diff --git a/.gitignore b/.gitignore index 1b23659..6c87765 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ output.txt output.yaml gosec-report.json govulncheck-report.json +gitleaks-report.json +security-report.json security-report.md gosec*.log pr.txt diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..d6132b4 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,15 @@ +# gitleaks configuration +# https://github.com/gitleaks/gitleaks +# +# Extends the built-in ruleset. Only allowlist overrides are defined here. + +[allowlist] + description = "Global allowlist for generated and report files" + paths = [ + '''gosec-report\.json$''', + '''govulncheck-report\.json$''', + '''security-report\.json$''', + '''security-report\.md$''', + '''output\.json$''', + '''gibidify\.json$''', + ] diff --git a/.go-version b/.go-version index b45fe31..198ec23 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.25.5 +1.25.6 diff --git a/cli/terminal_test_helpers.go b/cli/terminal_test_helpers.go deleted file mode 100644 index 62c9d11..0000000 --- a/cli/terminal_test_helpers.go +++ /dev/null @@ -1,68 +0,0 @@ -package cli - -import "testing" - -// terminalEnvSetup defines environment variables for terminal detection tests. -type terminalEnvSetup struct { - Term string - CI string - GitHubActions string - NoColor string - ForceColor string -} - -// apply sets up the environment variables using t.Setenv. -func (e terminalEnvSetup) apply(t *testing.T) { - t.Helper() - - // Always set all environment variables to ensure isolation - // Empty string explicitly unsets the variable in the test environment - t.Setenv("TERM", e.Term) - t.Setenv("CI", e.CI) - t.Setenv("GITHUB_ACTIONS", e.GitHubActions) - t.Setenv("NO_COLOR", e.NoColor) - t.Setenv("FORCE_COLOR", e.ForceColor) -} - -// Common terminal environment setups for reuse across tests. -var ( - envDefaultTerminal = terminalEnvSetup{ - Term: "xterm-256color", - CI: "", - NoColor: "", - ForceColor: "", - } - - envDumbTerminal = terminalEnvSetup{ - Term: "dumb", - } - - envCIWithoutGitHub = terminalEnvSetup{ - Term: "xterm", - CI: "true", - GitHubActions: "", - } - - envGitHubActions = terminalEnvSetup{ - Term: "xterm", - CI: "true", - GitHubActions: "true", - NoColor: "", - } - - envNoColor = terminalEnvSetup{ - Term: "xterm-256color", - CI: "", - NoColor: "1", - ForceColor: "", - } - - envForceColor = terminalEnvSetup{ - Term: "dumb", - ForceColor: "1", - } - - envEmptyTerm = terminalEnvSetup{ - Term: "", - } -) diff --git a/cli/test_constants.go b/cli/test_constants.go deleted file mode 100644 index 74dd3ee..0000000 --- a/cli/test_constants.go +++ /dev/null @@ -1,42 +0,0 @@ -package cli - -// Test constants to avoid duplication in test files. -// These constants are used across multiple test files in the cli package. -const ( - // Error messages - testErrFileNotFound = "file not found" - testErrPermissionDenied = "permission denied" - testErrInvalidFormat = "invalid format" - testErrOther = "other error" - testErrEncoding = "encoding error" - testErrSourceRequired = "source directory is required" - testErrPathTraversal = "path traversal attempt detected" - testPathTraversalPath = "../../../etc/passwd" - - // Suggestion messages - testSuggestionsHeader = "Suggestions:" - testSuggestCheckPerms = "Check file/directory permissions" - testSuggestVerifyPath = "Verify the path is correct" - testSuggestFormat = "Use a supported format: markdown, json, yaml" - testSuggestFormatEx = "Example: -format markdown" - testSuggestCheckArgs = "Check your command line arguments" - testSuggestHelp = "Run with --help for usage information" - testSuggestDiskSpace = "Verify available disk space" - testSuggestReduceConcur = "Try with -concurrency 1 to reduce resource usage" - - // UI test strings - testWithColors = "with colors" - testWithoutColors = "without colors" - testProcessingMsg = "Processing files" - - // Flag names - testFlagSource = "-source" - testFlagConcurrency = "-concurrency" - - // Test file paths - testFilePath1 = "/test/file1.go" - testFilePath2 = "/test/file2.go" - - // Output markers - testErrorSuffix = " Error" -) diff --git a/docs/plans/2026-02-01-replace-check-secrets-with-gitleaks-design.md b/docs/plans/2026-02-01-replace-check-secrets-with-gitleaks-design.md new file mode 100644 index 0000000..fdf02c2 --- /dev/null +++ b/docs/plans/2026-02-01-replace-check-secrets-with-gitleaks-design.md @@ -0,0 +1,45 @@ +# Replace check_secrets with gitleaks + +## Problem + +The `check_secrets` function in `scripts/security-scan.sh` uses hand-rolled regex +patterns that produce false positives. The pattern `key\s*[:=]\s*['"][^'"]{8,}['"]` +matches every `configKey: "backpressure.maxPendingFiles"` line in +`config/getters_test.go` (40+ matches), causing `make security-full` to fail. + +The git history check (`git log --oneline -10 | grep -i "key|token"`) also matches +on benign commit messages containing words like "key" or "token". + +## Decision + +Replace the custom `check_secrets` function with +[gitleaks](https://github.com/gitleaks/gitleaks), a widely adopted Go-based secret +scanner with built-in rules for AWS keys, GitHub tokens, private keys, high-entropy +strings, and more. + +## Approach + +- **Drop-in replacement**: Only the `check_secrets` function body changes. The + function signature and return behavior (0 = clean, 1 = findings) remain identical. +- **`go run` invocation**: Use `go run github.com/gitleaks/gitleaks/v8@latest` so + the tool is fetched automatically if not cached. No changes to `install-tools.sh`. +- **Working tree scan only**: Use `gitleaks dir` to scan current files. No git + history scanning (matches current script behavior scope). +- **Config file**: A `.gitleaks.toml` at the project root extends gitleaks' built-in + rules with an allowlist to suppress known false positives in test files. +- **CI unaffected**: `.github/workflows/security.yml` runs its own inline steps + (gosec, govulncheck, checkmake, shfmt, yamllint, Trivy) and does not call + `security-scan.sh` or `check_secrets`. + +## Files Changed + +| File | Change | +|------|--------| +| `scripts/security-scan.sh` | Replace `check_secrets` function body | +| `.gitleaks.toml` | New file -- gitleaks configuration with allowlist | + +## Verification + +```bash +make security-full # should pass end-to-end +``` diff --git a/fileproc/processor_test.go b/fileproc/processor_test.go index d3ed849..59886dd 100644 --- a/fileproc/processor_test.go +++ b/fileproc/processor_test.go @@ -31,54 +31,6 @@ func writeTempConfig(t *testing.T, content string) string { return dir } -// collectWriteRequests runs a processing function and collects all WriteRequests. -// This helper wraps the common pattern of channel + goroutine + WaitGroup. -func collectWriteRequests(t *testing.T, process func(ch chan fileproc.WriteRequest)) []fileproc.WriteRequest { - t.Helper() - - ch := make(chan fileproc.WriteRequest, 10) - - var wg sync.WaitGroup - wg.Go(func() { - defer close(ch) - process(ch) - }) - - results := make([]fileproc.WriteRequest, 0) - for req := range ch { - results = append(results, req) - } - wg.Wait() - - return results -} - -// collectWriteRequestsWithContext runs a processing function with context and collects all WriteRequests. -func collectWriteRequestsWithContext( - ctx context.Context, - t *testing.T, - process func(ctx context.Context, ch chan fileproc.WriteRequest) error, -) ([]fileproc.WriteRequest, error) { - t.Helper() - - ch := make(chan fileproc.WriteRequest, 10) - var processErr error - - var wg sync.WaitGroup - wg.Go(func() { - defer close(ch) - processErr = process(ctx, ch) - }) - - results := make([]fileproc.WriteRequest, 0) - for req := range ch { - results = append(results, req) - } - wg.Wait() - - return results, processErr -} - func TestProcessFile(t *testing.T) { // Reset and load default config to ensure proper file size limits testutil.ResetViperConfig(t, "") diff --git a/fileproc/test_constants.go b/fileproc/test_constants.go deleted file mode 100644 index 9ecdc5e..0000000 --- a/fileproc/test_constants.go +++ /dev/null @@ -1,12 +0,0 @@ -package fileproc - -// Test constants to avoid duplication in test files. -// These constants are used across multiple test files in the fileproc package. -const ( - // Backpressure configuration keys - testBackpressureEnabled = "backpressure.enabled" - testBackpressureMaxMemory = "backpressure.maxMemoryUsage" - testBackpressureMemoryCheck = "backpressure.memoryCheckInterval" - testBackpressureMaxFiles = "backpressure.maxPendingFiles" - testBackpressureMaxWrites = "backpressure.maxPendingWrites" -) diff --git a/go.mod b/go.mod index fc0367c..3c28c80 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/ivuorinen/gibidify go 1.25 +toolchain go1.25.6 + require ( github.com/fatih/color v1.18.0 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 @@ -14,7 +16,7 @@ require ( require ( github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect @@ -26,7 +28,7 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index d2eaa60..e8de4ac 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -37,12 +37,8 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= -github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= -github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -55,21 +51,17 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/scripts/install-tools.sh b/scripts/install-tools.sh index 98a2862..084f2c7 100755 --- a/scripts/install-tools.sh +++ b/scripts/install-tools.sh @@ -86,7 +86,7 @@ check_dependencies() { if ! command -v checkmake &>/dev/null; then print_warning "checkmake not found, installing..." - go install github.com/checkmake/checkmake/cmd/checkmake@v0.2.2 + go install github.com/mrtazz/checkmake/cmd/checkmake@v0.2.2 fi if ! command -v eclint &>/dev/null; then @@ -99,11 +99,6 @@ check_dependencies() { go install honnef.co/go/tools/cmd/staticcheck@v0.6.1 fi - if ! command -v yamllint &>/dev/null; then - print_warning "yamllint not found, installing..." - go install mvdan.cc/yaml/cmd/yaml-lint@v2.4.0 - fi - # Formatting tools if ! command -v gofumpt &>/dev/null; then diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 6459aa7..f86cdcb 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -63,44 +63,25 @@ run_security_lint() { fi } -# Check for potential secrets +# Check for potential secrets using gitleaks check_secrets() { - print_status "Scanning for potential secrets and sensitive data..." + print_status "Scanning for potential secrets and sensitive data with gitleaks..." - local secrets_found=false - - # Common secret patterns - local patterns=( - "password\s*[:=]\s*['\"][^'\"]{3,}['\"]" - "secret\s*[:=]\s*['\"][^'\"]{3,}['\"]" - "key\s*[:=]\s*['\"][^'\"]{8,}['\"]" - "token\s*[:=]\s*['\"][^'\"]{8,}['\"]" - "api_?key\s*[:=]\s*['\"][^'\"]{8,}['\"]" - "aws_?access_?key" - "aws_?secret" - "AKIA[0-9A-Z]{16}" # AWS Access Key pattern - "github_?token" - "private_?key" - ) - - for pattern in "${patterns[@]}"; do - if grep -r -i -E "$pattern" --include="*.go" . 2>/dev/null; then - print_warning "Potential secret pattern found: $pattern" - secrets_found=true - fi - done - - # Check git history for secrets (last 10 commits) - if git log --oneline -10 | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then - print_warning "Potential secrets mentioned in recent commit messages" - secrets_found=true - fi - - if [[ "$secrets_found" = true ]]; then - print_warning "Potential secrets detected. Please review manually." - return 1 + local gitleaks_report="gitleaks-report.json" + if go run github.com/zricethezav/gitleaks/v8@latest dir \ + --config .gitleaks.toml \ + --report-format json \ + --report-path "$gitleaks_report" \ + --no-banner \ + .; then + print_success "No secrets detected by gitleaks" + rm -f "$gitleaks_report" else - print_success "No obvious secrets detected" + print_error "Secrets detected by gitleaks!" + if [[ -f "$gitleaks_report" ]]; then + echo "Detailed report saved to $gitleaks_report" + fi + return 1 fi } @@ -235,11 +216,14 @@ check_yaml_files() { print_status "Checking YAML files..." if find . -name "*.yml" -o -name "*.yaml" -type f | head -1 | grep -q .; then - if yamllint -c .yamllint .; then + if command -v yamllint >/dev/null 2>&1; then + if ! yamllint -c .yamllint .; then + print_error "YAML file issues detected!" + return 1 + fi print_success "YAML files check passed" else - print_error "YAML file issues detected!" - return 1 + print_warning "yamllint not found, skipping YAML file check" fi else print_status "No YAML files found, skipping yamllint check" @@ -268,7 +252,7 @@ generate_report() { - checkmake (Makefile linting) - shfmt (Shell script formatting) - yamllint (YAML file validation) -- Custom secret detection +- gitleaks (Secret detection) - Custom network address detection - Docker security checks - File permission checks @@ -276,6 +260,7 @@ generate_report() { ### Files Generated - \`gosec-report.json\` - Detailed gosec security findings - \`govulncheck-report.json\` - Dependency vulnerability report +- \`gitleaks-report.json\` - Secret detection findings (if any) ### Recommendations 1. Review all security findings in the generated reports @@ -350,6 +335,7 @@ main() { print_status "Generated reports:" print_status "- gosec-report.json (if exists)" print_status "- govulncheck-report.json (if exists)" + print_status "- gitleaks-report.json (if exists)" print_status "- security-report.md" fi