fix: security issues and use gitleaks (#163)

* fix(tests): remove unused test constants and helpers

Delete dead test code that caused 41 staticcheck U1000 violations:
- cli/test_constants.go (25 unused constants)
- cli/terminal_test_helpers.go (unused type, method, 7 variables)
- fileproc/test_constants.go (5 unused constants)
- fileproc/processor_test.go (2 unused helper functions)

* fix(security): replace custom secret detection with gitleaks

The hand-rolled check_secrets regex patterns produced false positives
on configKey test values, causing make security-full to fail.

Replace with gitleaks via go run for proper secret detection with
built-in rules and allowlist support for generated report files.

* chore(deps): update dependencies and fix install-tools

Update Go module dependencies to latest versions.
Fix checkmake install path and remove yamllint go install
(yamllint is a Python tool, not installable via go install).

* docs: add design document for gitleaks integration

* feat: update go to 1.25.6
This commit is contained in:
2026-02-01 22:09:24 +02:00
committed by GitHub
parent 7a99534252
commit 994099137a
12 changed files with 100 additions and 233 deletions

2
.gitignore vendored
View File

@@ -14,6 +14,8 @@ output.txt
output.yaml
gosec-report.json
govulncheck-report.json
gitleaks-report.json
security-report.json
security-report.md
gosec*.log
pr.txt

15
.gitleaks.toml Normal file
View File

@@ -0,0 +1,15 @@
# gitleaks configuration
# https://github.com/gitleaks/gitleaks
#
# Extends the built-in ruleset. Only allowlist overrides are defined here.
[allowlist]
description = "Global allowlist for generated and report files"
paths = [
'''gosec-report\.json$''',
'''govulncheck-report\.json$''',
'''security-report\.json$''',
'''security-report\.md$''',
'''output\.json$''',
'''gibidify\.json$''',
]

View File

@@ -1 +1 @@
1.25.5
1.25.6

View File

@@ -1,68 +0,0 @@
package cli
import "testing"
// terminalEnvSetup defines environment variables for terminal detection tests.
type terminalEnvSetup struct {
Term string
CI string
GitHubActions string
NoColor string
ForceColor string
}
// apply sets up the environment variables using t.Setenv.
func (e terminalEnvSetup) apply(t *testing.T) {
t.Helper()
// Always set all environment variables to ensure isolation
// Empty string explicitly unsets the variable in the test environment
t.Setenv("TERM", e.Term)
t.Setenv("CI", e.CI)
t.Setenv("GITHUB_ACTIONS", e.GitHubActions)
t.Setenv("NO_COLOR", e.NoColor)
t.Setenv("FORCE_COLOR", e.ForceColor)
}
// Common terminal environment setups for reuse across tests.
var (
envDefaultTerminal = terminalEnvSetup{
Term: "xterm-256color",
CI: "",
NoColor: "",
ForceColor: "",
}
envDumbTerminal = terminalEnvSetup{
Term: "dumb",
}
envCIWithoutGitHub = terminalEnvSetup{
Term: "xterm",
CI: "true",
GitHubActions: "",
}
envGitHubActions = terminalEnvSetup{
Term: "xterm",
CI: "true",
GitHubActions: "true",
NoColor: "",
}
envNoColor = terminalEnvSetup{
Term: "xterm-256color",
CI: "",
NoColor: "1",
ForceColor: "",
}
envForceColor = terminalEnvSetup{
Term: "dumb",
ForceColor: "1",
}
envEmptyTerm = terminalEnvSetup{
Term: "",
}
)

View File

@@ -1,42 +0,0 @@
package cli
// Test constants to avoid duplication in test files.
// These constants are used across multiple test files in the cli package.
const (
// Error messages
testErrFileNotFound = "file not found"
testErrPermissionDenied = "permission denied"
testErrInvalidFormat = "invalid format"
testErrOther = "other error"
testErrEncoding = "encoding error"
testErrSourceRequired = "source directory is required"
testErrPathTraversal = "path traversal attempt detected"
testPathTraversalPath = "../../../etc/passwd"
// Suggestion messages
testSuggestionsHeader = "Suggestions:"
testSuggestCheckPerms = "Check file/directory permissions"
testSuggestVerifyPath = "Verify the path is correct"
testSuggestFormat = "Use a supported format: markdown, json, yaml"
testSuggestFormatEx = "Example: -format markdown"
testSuggestCheckArgs = "Check your command line arguments"
testSuggestHelp = "Run with --help for usage information"
testSuggestDiskSpace = "Verify available disk space"
testSuggestReduceConcur = "Try with -concurrency 1 to reduce resource usage"
// UI test strings
testWithColors = "with colors"
testWithoutColors = "without colors"
testProcessingMsg = "Processing files"
// Flag names
testFlagSource = "-source"
testFlagConcurrency = "-concurrency"
// Test file paths
testFilePath1 = "/test/file1.go"
testFilePath2 = "/test/file2.go"
// Output markers
testErrorSuffix = " Error"
)

View File

@@ -0,0 +1,45 @@
# Replace check_secrets with gitleaks
## Problem
The `check_secrets` function in `scripts/security-scan.sh` uses hand-rolled regex
patterns that produce false positives. The pattern `key\s*[:=]\s*['"][^'"]{8,}['"]`
matches every `configKey: "backpressure.maxPendingFiles"` line in
`config/getters_test.go` (40+ matches), causing `make security-full` to fail.
The git history check (`git log --oneline -10 | grep -i "key|token"`) also matches
on benign commit messages containing words like "key" or "token".
## Decision
Replace the custom `check_secrets` function with
[gitleaks](https://github.com/gitleaks/gitleaks), a widely adopted Go-based secret
scanner with built-in rules for AWS keys, GitHub tokens, private keys, high-entropy
strings, and more.
## Approach
- **Drop-in replacement**: Only the `check_secrets` function body changes. The
function signature and return behavior (0 = clean, 1 = findings) remain identical.
- **`go run` invocation**: Use `go run github.com/gitleaks/gitleaks/v8@latest` so
the tool is fetched automatically if not cached. No changes to `install-tools.sh`.
- **Working tree scan only**: Use `gitleaks dir` to scan current files. No git
history scanning (matches current script behavior scope).
- **Config file**: A `.gitleaks.toml` at the project root extends gitleaks' built-in
rules with an allowlist to suppress known false positives in test files.
- **CI unaffected**: `.github/workflows/security.yml` runs its own inline steps
(gosec, govulncheck, checkmake, shfmt, yamllint, Trivy) and does not call
`security-scan.sh` or `check_secrets`.
## Files Changed
| File | Change |
|------|--------|
| `scripts/security-scan.sh` | Replace `check_secrets` function body |
| `.gitleaks.toml` | New file -- gitleaks configuration with allowlist |
## Verification
```bash
make security-full # should pass end-to-end
```

View File

@@ -31,54 +31,6 @@ func writeTempConfig(t *testing.T, content string) string {
return dir
}
// collectWriteRequests runs a processing function and collects all WriteRequests.
// This helper wraps the common pattern of channel + goroutine + WaitGroup.
func collectWriteRequests(t *testing.T, process func(ch chan fileproc.WriteRequest)) []fileproc.WriteRequest {
t.Helper()
ch := make(chan fileproc.WriteRequest, 10)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
process(ch)
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
return results
}
// collectWriteRequestsWithContext runs a processing function with context and collects all WriteRequests.
func collectWriteRequestsWithContext(
ctx context.Context,
t *testing.T,
process func(ctx context.Context, ch chan fileproc.WriteRequest) error,
) ([]fileproc.WriteRequest, error) {
t.Helper()
ch := make(chan fileproc.WriteRequest, 10)
var processErr error
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
processErr = process(ctx, ch)
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
return results, processErr
}
func TestProcessFile(t *testing.T) {
// Reset and load default config to ensure proper file size limits
testutil.ResetViperConfig(t, "")

View File

@@ -1,12 +0,0 @@
package fileproc
// Test constants to avoid duplication in test files.
// These constants are used across multiple test files in the fileproc package.
const (
// Backpressure configuration keys
testBackpressureEnabled = "backpressure.enabled"
testBackpressureMaxMemory = "backpressure.maxMemoryUsage"
testBackpressureMemoryCheck = "backpressure.memoryCheckInterval"
testBackpressureMaxFiles = "backpressure.maxPendingFiles"
testBackpressureMaxWrites = "backpressure.maxPendingWrites"
)

8
go.mod
View File

@@ -2,6 +2,8 @@ module github.com/ivuorinen/gibidify
go 1.25
toolchain go1.25.6
require (
github.com/fatih/color v1.18.0
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
@@ -14,7 +16,7 @@ require (
require (
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
@@ -26,7 +28,7 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/term v0.39.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
)

20
go.sum
View File

@@ -9,8 +9,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -37,12 +37,8 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
@@ -55,21 +51,17 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@@ -86,7 +86,7 @@ check_dependencies() {
if ! command -v checkmake &>/dev/null; then
print_warning "checkmake not found, installing..."
go install github.com/checkmake/checkmake/cmd/checkmake@v0.2.2
go install github.com/mrtazz/checkmake/cmd/checkmake@v0.2.2
fi
if ! command -v eclint &>/dev/null; then
@@ -99,11 +99,6 @@ check_dependencies() {
go install honnef.co/go/tools/cmd/staticcheck@v0.6.1
fi
if ! command -v yamllint &>/dev/null; then
print_warning "yamllint not found, installing..."
go install mvdan.cc/yaml/cmd/yaml-lint@v2.4.0
fi
# Formatting tools
if ! command -v gofumpt &>/dev/null; then

View File

@@ -63,44 +63,25 @@ run_security_lint() {
fi
}
# Check for potential secrets
# Check for potential secrets using gitleaks
check_secrets() {
print_status "Scanning for potential secrets and sensitive data..."
print_status "Scanning for potential secrets and sensitive data with gitleaks..."
local secrets_found=false
# Common secret patterns
local patterns=(
"password\s*[:=]\s*['\"][^'\"]{3,}['\"]"
"secret\s*[:=]\s*['\"][^'\"]{3,}['\"]"
"key\s*[:=]\s*['\"][^'\"]{8,}['\"]"
"token\s*[:=]\s*['\"][^'\"]{8,}['\"]"
"api_?key\s*[:=]\s*['\"][^'\"]{8,}['\"]"
"aws_?access_?key"
"aws_?secret"
"AKIA[0-9A-Z]{16}" # AWS Access Key pattern
"github_?token"
"private_?key"
)
for pattern in "${patterns[@]}"; do
if grep -r -i -E "$pattern" --include="*.go" . 2>/dev/null; then
print_warning "Potential secret pattern found: $pattern"
secrets_found=true
fi
done
# Check git history for secrets (last 10 commits)
if git log --oneline -10 | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then
print_warning "Potential secrets mentioned in recent commit messages"
secrets_found=true
fi
if [[ "$secrets_found" = true ]]; then
print_warning "Potential secrets detected. Please review manually."
return 1
local gitleaks_report="gitleaks-report.json"
if go run github.com/zricethezav/gitleaks/v8@latest dir \
--config .gitleaks.toml \
--report-format json \
--report-path "$gitleaks_report" \
--no-banner \
.; then
print_success "No secrets detected by gitleaks"
rm -f "$gitleaks_report"
else
print_success "No obvious secrets detected"
print_error "Secrets detected by gitleaks!"
if [[ -f "$gitleaks_report" ]]; then
echo "Detailed report saved to $gitleaks_report"
fi
return 1
fi
}
@@ -235,11 +216,14 @@ check_yaml_files() {
print_status "Checking YAML files..."
if find . -name "*.yml" -o -name "*.yaml" -type f | head -1 | grep -q .; then
if yamllint -c .yamllint .; then
if command -v yamllint >/dev/null 2>&1; then
if ! yamllint -c .yamllint .; then
print_error "YAML file issues detected!"
return 1
fi
print_success "YAML files check passed"
else
print_error "YAML file issues detected!"
return 1
print_warning "yamllint not found, skipping YAML file check"
fi
else
print_status "No YAML files found, skipping yamllint check"
@@ -268,7 +252,7 @@ generate_report() {
- checkmake (Makefile linting)
- shfmt (Shell script formatting)
- yamllint (YAML file validation)
- Custom secret detection
- gitleaks (Secret detection)
- Custom network address detection
- Docker security checks
- File permission checks
@@ -276,6 +260,7 @@ generate_report() {
### Files Generated
- \`gosec-report.json\` - Detailed gosec security findings
- \`govulncheck-report.json\` - Dependency vulnerability report
- \`gitleaks-report.json\` - Secret detection findings (if any)
### Recommendations
1. Review all security findings in the generated reports
@@ -350,6 +335,7 @@ main() {
print_status "Generated reports:"
print_status "- gosec-report.json (if exists)"
print_status "- govulncheck-report.json (if exists)"
print_status "- gitleaks-report.json (if exists)"
print_status "- security-report.md"
fi