feat(security): improve security features, fixes

This commit is contained in:
2025-07-19 01:37:52 +03:00
parent e35126856d
commit b369d317b1
19 changed files with 2266 additions and 92 deletions

8
.checkmake Normal file
View File

@@ -0,0 +1,8 @@
# checkmake configuration
# See: https://github.com/mrtazz/checkmake#configuration
[rules.timestampexpansion]
disabled = true
[rules.maxbodylength]
disabled = true

142
.github/workflows/security.yml vendored Normal file
View File

@@ -0,0 +1,142 @@
name: Security Scan
on:
push:
branches: [main, develop]
pull_request:
branches: [main, develop]
schedule:
# Run security scan weekly on Sundays at 00:00 UTC
- cron: '0 0 * * 0'
permissions:
security-events: write
contents: read
actions: read
jobs:
security:
name: Security Analysis
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
- name: Cache Go modules
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
# Security Scanning with gosec
- name: Run gosec Security Scanner
uses: securecodewarrior/github-action-gosec@master
with:
args: '-fmt sarif -out gosec-results.sarif ./...'
- name: Upload gosec results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: gosec-results.sarif
# Dependency Vulnerability Scanning
- name: Run govulncheck
run: |
go install golang.org/x/vuln/cmd/govulncheck@latest
govulncheck -json ./... > govulncheck-results.json || true
- name: Parse govulncheck results
run: |
if [ -s govulncheck-results.json ]; then
echo "::warning::Vulnerability check completed. Check govulncheck-results.json for details."
if grep -q '"finding"' govulncheck-results.json; then
echo "::error::Vulnerabilities found in dependencies!"
cat govulncheck-results.json
exit 1
fi
fi
# Additional Security Linting
- name: Run security-focused golangci-lint
run: |
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
golangci-lint run --enable=gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused \
--timeout=5m
# Makefile Linting
- name: Run checkmake on Makefile
run: |
go install github.com/mrtazz/checkmake/cmd/checkmake@latest
checkmake --config=.checkmake Makefile
# Shell Script Formatting Check
- name: Check shell script formatting
run: |
go install mvdan.cc/sh/v3/cmd/shfmt@latest
shfmt -d .
# YAML Linting
- name: Run YAML linting
run: |
go install github.com/excilsploft/yamllint@latest
yamllint -c .yamllint .
# Secrets Detection (basic patterns)
- name: Run secrets detection
run: |
echo "Scanning for potential secrets..."
# Look for common secret patterns
git log --all --full-history -- . | grep -i -E "(password|secret|key|token|api_key)" || true
find . -type f -name "*.go" -exec grep -H -i -E "(password|secret|key|token|api_key)\s*[:=]" {} \; || true
# Check for hardcoded IPs and URLs
- name: Check for hardcoded network addresses
run: |
echo "Scanning for hardcoded network addresses..."
find . -type f -name "*.go" -exec grep -H -E "([0-9]{1,3}\.){3}[0-9]{1,3}" {} \; || true
find . -type f -name "*.go" -exec grep -H -E "https?://[^/\s]+" {} \; | \
grep -v "example.com|localhost|127.0.0.1" || true
# Docker Security (if Dockerfile exists)
- name: Run Docker security scan
if: hashFiles('Dockerfile') != ''
run: |
docker run --rm -v "$PWD":/workspace \
aquasec/trivy:latest fs --security-checks vuln,config /workspace/Dockerfile || true
# SAST with CodeQL (if available)
- name: Initialize CodeQL
if: github.event_name != 'schedule'
uses: github/codeql-action/init@v3
with:
languages: go
- name: Autobuild
if: github.event_name != 'schedule'
uses: github/codeql-action/autobuild@v3
- name: Perform CodeQL Analysis
if: github.event_name != 'schedule'
uses: github/codeql-action/analyze@v3
# Upload artifacts for review
- name: Upload security scan results
uses: actions/upload-artifact@v4
if: always()
with:
name: security-scan-results
path: |
gosec-results.sarif
govulncheck-results.json
retention-days: 30

View File

@@ -143,7 +143,7 @@ linters-settings:
# - insert_final_newline = true (enforced by gofumpt)
# - trim_trailing_whitespace = true (enforced by whitespace linter)
# - indent_style = tab, tab_width = 2 (enforced by gofumpt and lll)
whitespace:
multi-if: false # EditorConfig: trim trailing whitespace
multi-func: false # EditorConfig: trim trailing whitespace
@@ -198,18 +198,18 @@ issues:
max-issues-per-linter: 0
max-same-issues: 0
uniq-by-line: true
exclude-dirs:
- vendor
- third_party
- testdata
- examples
- .git
exclude-files:
- ".*\\.pb\\.go$"
- ".*\\.gen\\.go$"
exclude-rules:
- path: _test\.go
linters:
@@ -253,4 +253,4 @@ issues:
severity:
default-severity: error
case-sensitive: false
case-sensitive: false

40
.yamllint Normal file
View File

@@ -0,0 +1,40 @@
# yamllint configuration
# See: https://yamllint.readthedocs.io/en/stable/configuration.html
extends: default
# Ignore generated output files
ignore: |
gibidify.yaml
gibidify.yml
output.yaml
output.yml
rules:
# Allow longer lines for URLs and commands in GitHub Actions
line-length:
max: 120
level: warning
# Allow 2-space indentation to match EditorConfig
indentation:
spaces: 2
indent-sequences: true
check-multi-line-strings: false
# Allow truthy values like 'on' in GitHub Actions
truthy:
allowed-values: ['true', 'false', 'on', 'off']
check-keys: false
# Allow empty values in YAML
empty-values:
forbid-in-block-mappings: false
forbid-in-flow-mappings: false
# Relax comments formatting
comments:
min-spaces-from-content: 1
# Allow document start marker to be optional
document-start: disable

View File

@@ -1,4 +1,4 @@
.PHONY: help install-tools lint lint-fix lint-verbose test coverage build clean all build-benchmark benchmark benchmark-collection benchmark-processing benchmark-concurrency benchmark-format
.PHONY: help install-tools lint lint-fix lint-verbose test coverage build clean all build-benchmark benchmark benchmark-collection benchmark-processing benchmark-concurrency benchmark-format security security-full vuln-check check-all dev-setup
# Default target shows help
.DEFAULT_GOAL := help
@@ -6,28 +6,9 @@
# All target runs full workflow
all: lint test build
# Help target
# Help target
help:
@echo "Available targets:"
@echo " install-tools - Install required linting and development tools"
@echo " lint - Run all linters"
@echo " lint-fix - Run linters with auto-fix enabled"
@echo " lint-verbose - Run linters with verbose output"
@echo " test - Run tests"
@echo " coverage - Run tests with coverage"
@echo " build - Build the application"
@echo " clean - Clean build artifacts"
@echo " all - Run lint, test, and build"
@echo ""
@echo "Benchmark targets:"
@echo " build-benchmark - Build the benchmark binary"
@echo " benchmark - Run all benchmarks"
@echo " benchmark-collection - Run file collection benchmarks"
@echo " benchmark-processing - Run file processing benchmarks"
@echo " benchmark-concurrency - Run concurrency benchmarks"
@echo " benchmark-format - Run format benchmarks"
@echo ""
@echo "Run 'make <target>' to execute a specific target."
@cat scripts/help.txt
# Install required tools
install-tools:
@@ -43,12 +24,17 @@ install-tools:
@go install github.com/securego/gosec/v2/cmd/gosec@latest
@echo "Installing gocyclo..."
@go install github.com/fzipp/gocyclo/cmd/gocyclo@latest
@echo "Installing checkmake..."
@go install github.com/mrtazz/checkmake/cmd/checkmake@latest
@echo "Installing shfmt..."
@go install mvdan.cc/sh/v3/cmd/shfmt@latest
@echo "Installing yamllint (Go-based)..."
@go install github.com/excilsploft/yamllint@latest
@echo "All tools installed successfully!"
# Run linters
lint:
@echo "Running golangci-lint..."
@golangci-lint run ./...
@./scripts/lint.sh
# Run linters with auto-fix
lint-fix:
@@ -60,14 +46,27 @@ lint-fix:
@go fmt ./...
@echo "Running go mod tidy..."
@go mod tidy
@echo "Running shfmt formatting..."
@shfmt -w -i 2 -ci .
@echo "Running golangci-lint with --fix..."
@golangci-lint run --fix ./...
@echo "Auto-fix completed. Running final lint check..."
@golangci-lint run ./...
@echo "Running checkmake..."
@checkmake --config=.checkmake Makefile
@echo "Running yamllint..."
@yamllint -c .yamllint .
# Run linters with verbose output
lint-verbose:
@echo "Running golangci-lint (verbose)..."
@golangci-lint run -v ./...
@echo "Running checkmake (verbose)..."
@checkmake --config=.checkmake --format="{{.Line}}:{{.Rule}}:{{.Violation}}" Makefile
@echo "Running shfmt check (verbose)..."
@shfmt -d .
@echo "Running yamllint (verbose)..."
@yamllint -c .yamllint -f parsable .
# Run tests
test:
@@ -129,4 +128,20 @@ benchmark-concurrency: build-benchmark
benchmark-format: build-benchmark
@echo "Running format benchmarks..."
@./gibidify-benchmark -type=format
@./gibidify-benchmark -type=format
# Security targets
security:
@echo "Running comprehensive security scan..."
@./scripts/security-scan.sh
security-full:
@echo "Running full security analysis..."
@./scripts/security-scan.sh
@echo "Running additional security checks..."
@golangci-lint run --enable-all --disable=depguard,exhaustruct,ireturn,varnamelen,wrapcheck --timeout=10m
vuln-check:
@echo "Checking for dependency vulnerabilities..."
@go install golang.org/x/vuln/cmd/govulncheck@latest
@govulncheck ./...

View File

@@ -66,6 +66,11 @@ func (f *Flags) validate() error {
return NewCLIMissingSourceError()
}
// Validate source path for security
if err := utils.ValidateSourcePath(f.SourceDir); err != nil {
return err
}
// Validate output format
if err := config.ValidateOutputFormat(f.Format); err != nil {
return err
@@ -89,5 +94,11 @@ func (f *Flags) setDefaultDestination() error {
baseName := utils.GetBaseName(absRoot)
f.Destination = baseName + "." + f.Format
}
// Validate destination path for security
if err := utils.ValidateDestinationPath(f.Destination); err != nil {
return err
}
return nil
}

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"fmt"
"os"
"sync"
@@ -14,9 +15,10 @@ import (
// Processor handles the main file processing logic.
type Processor struct {
flags *Flags
backpressure *fileproc.BackpressureManager
ui *UIManager
flags *Flags
backpressure *fileproc.BackpressureManager
resourceMonitor *fileproc.ResourceMonitor
ui *UIManager
}
// NewProcessor creates a new processor with the given flags.
@@ -28,14 +30,19 @@ func NewProcessor(flags *Flags) *Processor {
ui.SetProgressOutput(!flags.NoProgress)
return &Processor{
flags: flags,
backpressure: fileproc.NewBackpressureManager(),
ui: ui,
flags: flags,
backpressure: fileproc.NewBackpressureManager(),
resourceMonitor: fileproc.NewResourceMonitor(),
ui: ui,
}
}
// Process executes the main file processing workflow.
func (p *Processor) Process(ctx context.Context) error {
// Create overall processing context with timeout
overallCtx, overallCancel := p.resourceMonitor.CreateOverallProcessingContext(ctx)
defer overallCancel()
// Configure file type registry
p.configureFileTypes()
@@ -46,6 +53,10 @@ func (p *Processor) Process(ctx context.Context) error {
p.ui.PrintInfo("Destination: %s", p.flags.Destination)
p.ui.PrintInfo("Workers: %d", p.flags.Concurrency)
// Log resource monitoring configuration
p.resourceMonitor.LogResourceInfo()
p.backpressure.LogBackpressureInfo()
// Collect files with progress indication
p.ui.PrintInfo("📁 Collecting files...")
files, err := p.collectFiles()
@@ -56,8 +67,13 @@ func (p *Processor) Process(ctx context.Context) error {
// Show collection results
p.ui.PrintSuccess("Found %d files to process", len(files))
// Process files
return p.processFiles(ctx, files)
// Pre-validate file collection against resource limits
if err := p.validateFileCollection(files); err != nil {
return err
}
// Process files with overall timeout
return p.processFiles(overallCtx, files)
}
// configureFileTypes configures the file type registry.
@@ -84,6 +100,61 @@ func (p *Processor) collectFiles() ([]string, error) {
return files, nil
}
// validateFileCollection validates the collected files against resource limits.
func (p *Processor) validateFileCollection(files []string) error {
if !config.GetResourceLimitsEnabled() {
return nil
}
// Check file count limit
maxFiles := config.GetMaxFiles()
if len(files) > maxFiles {
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitFiles,
fmt.Sprintf("file count (%d) exceeds maximum limit (%d)", len(files), maxFiles),
"",
map[string]interface{}{
"file_count": len(files),
"max_files": maxFiles,
},
)
}
// Check total size limit (estimate)
maxTotalSize := config.GetMaxTotalSize()
totalSize := int64(0)
oversizedFiles := 0
for _, filePath := range files {
if fileInfo, err := os.Stat(filePath); err == nil {
totalSize += fileInfo.Size()
if totalSize > maxTotalSize {
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitTotalSize,
fmt.Sprintf("total file size (%d bytes) would exceed maximum limit (%d bytes)", totalSize, maxTotalSize),
"",
map[string]interface{}{
"total_size": totalSize,
"max_total_size": maxTotalSize,
"files_checked": len(files),
},
)
}
} else {
oversizedFiles++
}
}
if oversizedFiles > 0 {
logrus.Warnf("Could not stat %d files during pre-validation", oversizedFiles)
}
logrus.Infof("Pre-validation passed: %d files, %d MB total", len(files), totalSize/1024/1024)
return nil
}
// processFiles processes the collected files.
func (p *Processor) processFiles(ctx context.Context, files []string) error {
outFile, err := p.createOutputFile()
@@ -127,7 +198,8 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error {
// createOutputFile creates the output file.
func (p *Processor) createOutputFile() (*os.File, error) {
outFile, err := os.Create(p.flags.Destination) // #nosec G304 - destination is user-provided CLI arg
// Destination path has been validated in CLI flags validation for path traversal attempts
outFile, err := os.Create(p.flags.Destination) // #nosec G304 - destination is validated in flags.validate()
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileCreate, "failed to create output file").WithFilePath(p.flags.Destination)
}
@@ -153,19 +225,27 @@ func (p *Processor) worker(ctx context.Context, wg *sync.WaitGroup, fileCh chan
if !ok {
return
}
p.processFile(filePath, writeCh)
p.processFile(ctx, filePath, writeCh)
}
}
}
// processFile processes a single file.
func (p *Processor) processFile(filePath string, writeCh chan fileproc.WriteRequest) {
// processFile processes a single file with resource monitoring.
func (p *Processor) processFile(ctx context.Context, filePath string, writeCh chan fileproc.WriteRequest) {
// Check for emergency stop
if p.resourceMonitor.IsEmergencyStopActive() {
logrus.Warnf("Emergency stop active, skipping file: %s", filePath)
return
}
absRoot, err := utils.GetAbsolutePath(p.flags.SourceDir)
if err != nil {
utils.LogError("Failed to get absolute path", err)
return
}
fileproc.ProcessFile(filePath, writeCh, absRoot)
// Use the resource monitor-aware processing
fileproc.ProcessFileWithMonitor(ctx, filePath, writeCh, absRoot, p.resourceMonitor)
// Update progress bar
p.ui.UpdateProgress(1)
@@ -200,11 +280,35 @@ func (p *Processor) waitForCompletion(wg *sync.WaitGroup, writeCh chan fileproc.
<-writerDone
}
// logFinalStats logs the final back-pressure statistics.
// logFinalStats logs the final back-pressure and resource monitoring statistics.
func (p *Processor) logFinalStats() {
stats := p.backpressure.GetStats()
if stats.Enabled {
// Log back-pressure stats
backpressureStats := p.backpressure.GetStats()
if backpressureStats.Enabled {
logrus.Infof("Back-pressure stats: processed=%d files, memory=%dMB/%dMB",
stats.FilesProcessed, stats.CurrentMemoryUsage/1024/1024, stats.MaxMemoryUsage/1024/1024)
backpressureStats.FilesProcessed, backpressureStats.CurrentMemoryUsage/1024/1024, backpressureStats.MaxMemoryUsage/1024/1024)
}
// Log resource monitoring stats
resourceStats := p.resourceMonitor.GetMetrics()
if config.GetResourceLimitsEnabled() {
logrus.Infof("Resource stats: processed=%d files, totalSize=%dMB, avgFileSize=%.2fKB, rate=%.2f files/sec",
resourceStats.FilesProcessed, resourceStats.TotalSizeProcessed/1024/1024,
resourceStats.AverageFileSize/1024, resourceStats.ProcessingRate)
if len(resourceStats.ViolationsDetected) > 0 {
logrus.Warnf("Resource violations detected: %v", resourceStats.ViolationsDetected)
}
if resourceStats.DegradationActive {
logrus.Warnf("Processing completed with degradation mode active")
}
if resourceStats.EmergencyStopActive {
logrus.Errorf("Processing completed with emergency stop active")
}
}
// Clean up resource monitor
p.resourceMonitor.Close()
}

View File

@@ -25,20 +25,20 @@ ignoreDirectories:
fileTypes:
# Enable/disable file type detection entirely (default: true)
enabled: true
# Add custom image extensions
customImageExtensions:
- .webp
- .avif
- .heic
- .jxl
# Add custom binary extensions
customBinaryExtensions:
- .custom
- .proprietary
- .blob
# Add custom language mappings
customLanguages:
.zig: zig
@@ -51,17 +51,17 @@ fileTypes:
.fennel: fennel
.wast: wast
.wat: wat
# Disable specific default image extensions
disabledImageExtensions:
- .bmp # Disable bitmap support
- .tif # Disable TIFF support
# Disable specific default binary extensions
disabledBinaryExtensions:
- .exe # Don't treat executables as binary
- .dll # Don't treat DLL files as binary
# Disable specific default language extensions
disabledLanguageExtensions:
- .bat # Don't detect batch files
@@ -81,4 +81,4 @@ filePatterns:
- "*.go"
- "*.py"
- "*.js"
- "*.ts"
- "*.ts"

79
config.yaml.example Normal file
View File

@@ -0,0 +1,79 @@
# Gibidify Configuration Example
# This file demonstrates all available configuration options
# File size limit for individual files (in bytes)
# Default: 5242880 (5MB), Min: 1024 (1KB), Max: 104857600 (100MB)
fileSizeLimit: 5242880
# Directories to ignore during traversal
ignoreDirectories:
- vendor
- node_modules
- .git
- dist
- build
- target
- bower_components
- cache
- tmp
# File type detection and filtering
fileTypes:
enabled: true
customImageExtensions: []
customBinaryExtensions: []
customLanguages: {}
disabledImageExtensions: []
disabledBinaryExtensions: []
disabledLanguageExtensions: []
# Back-pressure management for memory optimization
backpressure:
enabled: true
maxPendingFiles: 1000 # Max files in channel buffer
maxPendingWrites: 100 # Max writes in channel buffer
maxMemoryUsage: 104857600 # 100MB soft memory limit
memoryCheckInterval: 1000 # Check memory every N files
# Resource limits for DoS protection and security
resourceLimits:
enabled: true
# File processing limits
maxFiles: 10000 # Maximum number of files to process
maxTotalSize: 1073741824 # Maximum total size (1GB)
# Timeout limits (in seconds)
fileProcessingTimeoutSec: 30 # Timeout for individual file processing
overallTimeoutSec: 3600 # Overall processing timeout (1 hour)
# Concurrency limits
maxConcurrentReads: 10 # Maximum concurrent file reading operations
# Rate limiting (0 = disabled)
rateLimitFilesPerSec: 0 # Files per second rate limit
# Memory limits
hardMemoryLimitMB: 512 # Hard memory limit (512MB)
# Safety features
enableGracefulDegradation: true # Enable graceful degradation on resource pressure
enableResourceMonitoring: true # Enable detailed resource monitoring
# Optional: Maximum concurrency for workers
# Default: number of CPU cores
# maxConcurrency: 4
# Optional: Supported output formats
# Default: ["json", "yaml", "markdown"]
# supportedFormats:
# - json
# - yaml
# - markdown
# Optional: File patterns to include
# Default: all files (empty list means no pattern filtering)
# filePatterns:
# - "*.go"
# - "*.py"
# - "*.js"

View File

@@ -20,6 +20,57 @@ const (
MinFileSizeLimit = 1024
// MaxFileSizeLimit is the maximum allowed file size limit (100MB).
MaxFileSizeLimit = 104857600
// Resource Limit Constants
// DefaultMaxFiles is the default maximum number of files to process.
DefaultMaxFiles = 10000
// MinMaxFiles is the minimum allowed file count limit.
MinMaxFiles = 1
// MaxMaxFiles is the maximum allowed file count limit.
MaxMaxFiles = 1000000
// DefaultMaxTotalSize is the default maximum total size of files (1GB).
DefaultMaxTotalSize = 1073741824
// MinMaxTotalSize is the minimum allowed total size limit (1MB).
MinMaxTotalSize = 1048576
// MaxMaxTotalSize is the maximum allowed total size limit (100GB).
MaxMaxTotalSize = 107374182400
// DefaultFileProcessingTimeoutSec is the default timeout for individual file processing (30 seconds).
DefaultFileProcessingTimeoutSec = 30
// MinFileProcessingTimeoutSec is the minimum allowed file processing timeout (1 second).
MinFileProcessingTimeoutSec = 1
// MaxFileProcessingTimeoutSec is the maximum allowed file processing timeout (300 seconds).
MaxFileProcessingTimeoutSec = 300
// DefaultOverallTimeoutSec is the default timeout for overall processing (3600 seconds = 1 hour).
DefaultOverallTimeoutSec = 3600
// MinOverallTimeoutSec is the minimum allowed overall timeout (10 seconds).
MinOverallTimeoutSec = 10
// MaxOverallTimeoutSec is the maximum allowed overall timeout (86400 seconds = 24 hours).
MaxOverallTimeoutSec = 86400
// DefaultMaxConcurrentReads is the default maximum concurrent file reading operations.
DefaultMaxConcurrentReads = 10
// MinMaxConcurrentReads is the minimum allowed concurrent reads.
MinMaxConcurrentReads = 1
// MaxMaxConcurrentReads is the maximum allowed concurrent reads.
MaxMaxConcurrentReads = 100
// DefaultRateLimitFilesPerSec is the default rate limit for file processing (0 = disabled).
DefaultRateLimitFilesPerSec = 0
// MinRateLimitFilesPerSec is the minimum rate limit.
MinRateLimitFilesPerSec = 0
// MaxRateLimitFilesPerSec is the maximum rate limit.
MaxRateLimitFilesPerSec = 10000
// DefaultHardMemoryLimitMB is the default hard memory limit (512MB).
DefaultHardMemoryLimitMB = 512
// MinHardMemoryLimitMB is the minimum hard memory limit (64MB).
MinHardMemoryLimitMB = 64
// MaxHardMemoryLimitMB is the maximum hard memory limit (8192MB = 8GB).
MaxHardMemoryLimitMB = 8192
)
// LoadConfig reads configuration from a YAML file.
@@ -32,7 +83,13 @@ func LoadConfig() {
viper.SetConfigType("yaml")
if xdgConfig := os.Getenv("XDG_CONFIG_HOME"); xdgConfig != "" {
viper.AddConfigPath(filepath.Join(xdgConfig, "gibidify"))
// Validate XDG_CONFIG_HOME for path traversal attempts
if err := utils.ValidateConfigPath(xdgConfig); err != nil {
logrus.Warnf("Invalid XDG_CONFIG_HOME path, using default config: %v", err)
} else {
configPath := filepath.Join(xdgConfig, "gibidify")
viper.AddConfigPath(configPath)
}
} else if home, err := os.UserHomeDir(); err == nil {
viper.AddConfigPath(filepath.Join(home, ".config", "gibidify"))
}
@@ -81,6 +138,18 @@ func setDefaultConfig() {
viper.SetDefault("backpressure.maxPendingWrites", 100) // Max writes in write channel buffer
viper.SetDefault("backpressure.maxMemoryUsage", 104857600) // 100MB max memory usage
viper.SetDefault("backpressure.memoryCheckInterval", 1000) // Check memory every 1000 files
// Resource limit defaults
viper.SetDefault("resourceLimits.enabled", true)
viper.SetDefault("resourceLimits.maxFiles", DefaultMaxFiles)
viper.SetDefault("resourceLimits.maxTotalSize", DefaultMaxTotalSize)
viper.SetDefault("resourceLimits.fileProcessingTimeoutSec", DefaultFileProcessingTimeoutSec)
viper.SetDefault("resourceLimits.overallTimeoutSec", DefaultOverallTimeoutSec)
viper.SetDefault("resourceLimits.maxConcurrentReads", DefaultMaxConcurrentReads)
viper.SetDefault("resourceLimits.rateLimitFilesPerSec", DefaultRateLimitFilesPerSec)
viper.SetDefault("resourceLimits.hardMemoryLimitMB", DefaultHardMemoryLimitMB)
viper.SetDefault("resourceLimits.enableGracefulDegradation", true)
viper.SetDefault("resourceLimits.enableResourceMonitoring", true)
}
// GetFileSizeLimit returns the file size limit from configuration.
@@ -249,12 +318,85 @@ func ValidateConfig() error {
}
}
// Validate resource limits configuration
if viper.IsSet("resourceLimits.maxFiles") {
maxFiles := viper.GetInt("resourceLimits.maxFiles")
if maxFiles < MinMaxFiles {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxFiles (%d) must be at least %d", maxFiles, MinMaxFiles))
}
if maxFiles > MaxMaxFiles {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxFiles (%d) exceeds maximum (%d)", maxFiles, MaxMaxFiles))
}
}
if viper.IsSet("resourceLimits.maxTotalSize") {
maxTotalSize := viper.GetInt64("resourceLimits.maxTotalSize")
if maxTotalSize < MinMaxTotalSize {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxTotalSize (%d) must be at least %d", maxTotalSize, MinMaxTotalSize))
}
if maxTotalSize > MaxMaxTotalSize {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxTotalSize (%d) exceeds maximum (%d)", maxTotalSize, MaxMaxTotalSize))
}
}
if viper.IsSet("resourceLimits.fileProcessingTimeoutSec") {
timeout := viper.GetInt("resourceLimits.fileProcessingTimeoutSec")
if timeout < MinFileProcessingTimeoutSec {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.fileProcessingTimeoutSec (%d) must be at least %d", timeout, MinFileProcessingTimeoutSec))
}
if timeout > MaxFileProcessingTimeoutSec {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.fileProcessingTimeoutSec (%d) exceeds maximum (%d)", timeout, MaxFileProcessingTimeoutSec))
}
}
if viper.IsSet("resourceLimits.overallTimeoutSec") {
timeout := viper.GetInt("resourceLimits.overallTimeoutSec")
if timeout < MinOverallTimeoutSec {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) must be at least %d", timeout, MinOverallTimeoutSec))
}
if timeout > MaxOverallTimeoutSec {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) exceeds maximum (%d)", timeout, MaxOverallTimeoutSec))
}
}
if viper.IsSet("resourceLimits.maxConcurrentReads") {
maxReads := viper.GetInt("resourceLimits.maxConcurrentReads")
if maxReads < MinMaxConcurrentReads {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) must be at least %d", maxReads, MinMaxConcurrentReads))
}
if maxReads > MaxMaxConcurrentReads {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) exceeds maximum (%d)", maxReads, MaxMaxConcurrentReads))
}
}
if viper.IsSet("resourceLimits.rateLimitFilesPerSec") {
rateLimit := viper.GetInt("resourceLimits.rateLimitFilesPerSec")
if rateLimit < MinRateLimitFilesPerSec {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) must be at least %d", rateLimit, MinRateLimitFilesPerSec))
}
if rateLimit > MaxRateLimitFilesPerSec {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) exceeds maximum (%d)", rateLimit, MaxRateLimitFilesPerSec))
}
}
if viper.IsSet("resourceLimits.hardMemoryLimitMB") {
memLimit := viper.GetInt("resourceLimits.hardMemoryLimitMB")
if memLimit < MinHardMemoryLimitMB {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) must be at least %d", memLimit, MinHardMemoryLimitMB))
}
if memLimit > MaxHardMemoryLimitMB {
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) exceeds maximum (%d)", memLimit, MaxHardMemoryLimitMB))
}
}
if len(validationErrors) > 0 {
return utils.NewStructuredError(
utils.ErrorTypeConfiguration,
utils.CodeConfigValidation,
"configuration validation failed: "+strings.Join(validationErrors, "; "),
).WithContext("validation_errors", validationErrors)
"",
map[string]interface{}{"validation_errors": validationErrors},
)
}
return nil
@@ -290,7 +432,9 @@ func ValidateFileSize(size int64) error {
utils.ErrorTypeValidation,
utils.CodeValidationSize,
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", size, limit),
).WithContext("file_size", size).WithContext("size_limit", limit)
"",
map[string]interface{}{"file_size": size, "size_limit": limit},
)
}
return nil
}
@@ -302,7 +446,9 @@ func ValidateOutputFormat(format string) error {
utils.ErrorTypeValidation,
utils.CodeValidationFormat,
fmt.Sprintf("unsupported output format: %s (supported: json, yaml, markdown)", format),
).WithContext("format", format)
"",
map[string]interface{}{"format": format},
)
}
return nil
}
@@ -314,7 +460,9 @@ func ValidateConcurrency(concurrency int) error {
utils.ErrorTypeValidation,
utils.CodeValidationFormat,
fmt.Sprintf("concurrency (%d) must be at least 1", concurrency),
).WithContext("concurrency", concurrency)
"",
map[string]interface{}{"concurrency": concurrency},
)
}
if viper.IsSet("maxConcurrency") {
@@ -324,7 +472,9 @@ func ValidateConcurrency(concurrency int) error {
utils.ErrorTypeValidation,
utils.CodeValidationFormat,
fmt.Sprintf("concurrency (%d) exceeds maximum (%d)", concurrency, maxConcurrency),
).WithContext("concurrency", concurrency).WithContext("max_concurrency", maxConcurrency)
"",
map[string]interface{}{"concurrency": concurrency, "max_concurrency": maxConcurrency},
)
}
}
@@ -392,3 +542,55 @@ func GetMaxMemoryUsage() int64 {
func GetMemoryCheckInterval() int {
return viper.GetInt("backpressure.memoryCheckInterval")
}
// Resource Limit Configuration Getters
// GetResourceLimitsEnabled returns whether resource limits are enabled.
func GetResourceLimitsEnabled() bool {
return viper.GetBool("resourceLimits.enabled")
}
// GetMaxFiles returns the maximum number of files that can be processed.
func GetMaxFiles() int {
return viper.GetInt("resourceLimits.maxFiles")
}
// GetMaxTotalSize returns the maximum total size of files that can be processed.
func GetMaxTotalSize() int64 {
return viper.GetInt64("resourceLimits.maxTotalSize")
}
// GetFileProcessingTimeoutSec returns the timeout for individual file processing in seconds.
func GetFileProcessingTimeoutSec() int {
return viper.GetInt("resourceLimits.fileProcessingTimeoutSec")
}
// GetOverallTimeoutSec returns the timeout for overall processing in seconds.
func GetOverallTimeoutSec() int {
return viper.GetInt("resourceLimits.overallTimeoutSec")
}
// GetMaxConcurrentReads returns the maximum number of concurrent file reading operations.
func GetMaxConcurrentReads() int {
return viper.GetInt("resourceLimits.maxConcurrentReads")
}
// GetRateLimitFilesPerSec returns the rate limit for file processing (files per second).
func GetRateLimitFilesPerSec() int {
return viper.GetInt("resourceLimits.rateLimitFilesPerSec")
}
// GetHardMemoryLimitMB returns the hard memory limit in megabytes.
func GetHardMemoryLimitMB() int {
return viper.GetInt("resourceLimits.hardMemoryLimitMB")
}
// GetEnableGracefulDegradation returns whether graceful degradation is enabled.
func GetEnableGracefulDegradation() bool {
return viper.GetBool("resourceLimits.enableGracefulDegradation")
}
// GetEnableResourceMonitoring returns whether resource monitoring is enabled.
func GetEnableResourceMonitoring() bool {
return viper.GetBool("resourceLimits.enableResourceMonitoring")
}

View File

@@ -2,11 +2,15 @@
package fileproc
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/utils"
@@ -31,15 +35,26 @@ type WriteRequest struct {
// FileProcessor handles file processing operations.
type FileProcessor struct {
rootPath string
sizeLimit int64
rootPath string
sizeLimit int64
resourceMonitor *ResourceMonitor
}
// NewFileProcessor creates a new file processor.
func NewFileProcessor(rootPath string) *FileProcessor {
return &FileProcessor{
rootPath: rootPath,
sizeLimit: config.GetFileSizeLimit(),
rootPath: rootPath,
sizeLimit: config.GetFileSizeLimit(),
resourceMonitor: NewResourceMonitor(),
}
}
// NewFileProcessorWithMonitor creates a new file processor with a shared resource monitor.
func NewFileProcessorWithMonitor(rootPath string, monitor *ResourceMonitor) *FileProcessor {
return &FileProcessor{
rootPath: rootPath,
sizeLimit: config.GetFileSizeLimit(),
resourceMonitor: monitor,
}
}
@@ -47,30 +62,92 @@ func NewFileProcessor(rootPath string) *FileProcessor {
// It automatically chooses between loading the entire file or streaming based on file size.
func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) {
processor := NewFileProcessor(rootPath)
processor.Process(filePath, outCh)
ctx := context.Background()
processor.ProcessWithContext(ctx, filePath, outCh)
}
// ProcessFileWithMonitor processes a file using a shared resource monitor.
func ProcessFileWithMonitor(ctx context.Context, filePath string, outCh chan<- WriteRequest, rootPath string, monitor *ResourceMonitor) {
processor := NewFileProcessorWithMonitor(rootPath, monitor)
processor.ProcessWithContext(ctx, filePath, outCh)
}
// Process handles file processing with the configured settings.
func (p *FileProcessor) Process(filePath string, outCh chan<- WriteRequest) {
// Validate file
fileInfo, err := p.validateFile(filePath)
ctx := context.Background()
p.ProcessWithContext(ctx, filePath, outCh)
}
// ProcessWithContext handles file processing with context and resource monitoring.
func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, outCh chan<- WriteRequest) {
// Create file processing context with timeout
fileCtx, fileCancel := p.resourceMonitor.CreateFileProcessingContext(ctx)
defer fileCancel()
// Wait for rate limiting
if err := p.resourceMonitor.WaitForRateLimit(fileCtx); err != nil {
if err == context.DeadlineExceeded {
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing timeout during rate limiting", filePath, nil),
"File processing timeout during rate limiting: %s", filePath,
)
}
return
}
// Validate file and check resource limits
fileInfo, err := p.validateFileWithLimits(fileCtx, filePath)
if err != nil {
return // Error already logged
}
// Acquire read slot for concurrent processing
if err := p.resourceMonitor.AcquireReadSlot(fileCtx); err != nil {
if err == context.DeadlineExceeded {
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing timeout waiting for read slot", filePath, nil),
"File processing timeout waiting for read slot: %s", filePath,
)
}
return
}
defer p.resourceMonitor.ReleaseReadSlot()
// Check hard memory limits before processing
if err := p.resourceMonitor.CheckHardMemoryLimit(); err != nil {
utils.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath)
return
}
// Get relative path
relPath := p.getRelativePath(filePath)
// Process file with timeout
processStart := time.Now()
defer func() {
// Record successful processing
p.resourceMonitor.RecordFileProcessed(fileInfo.Size())
logrus.Debugf("File processed in %v: %s", time.Since(processStart), filePath)
}()
// Choose processing strategy based on file size
if fileInfo.Size() <= StreamThreshold {
p.processInMemory(filePath, relPath, outCh)
p.processInMemoryWithContext(fileCtx, filePath, relPath, outCh)
} else {
p.processStreaming(filePath, relPath, outCh)
p.processStreamingWithContext(fileCtx, filePath, relPath, outCh)
}
}
// validateFile checks if the file can be processed.
func (p *FileProcessor) validateFile(filePath string) (os.FileInfo, error) {
// validateFileWithLimits checks if the file can be processed with resource limits.
func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath string) (os.FileInfo, error) {
// Check context cancellation
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
fileInfo, err := os.Stat(filePath)
if err != nil {
structErr := utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to stat file").WithFilePath(filePath)
@@ -78,19 +155,31 @@ func (p *FileProcessor) validateFile(filePath string) (os.FileInfo, error) {
return nil, err
}
// Check size limit
// Check traditional size limit
if fileInfo.Size() > p.sizeLimit {
context := map[string]interface{}{
"file_size": fileInfo.Size(),
"size_limit": p.sizeLimit,
}
utils.LogErrorf(
utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeValidationSize,
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", fileInfo.Size(), p.sizeLimit),
).WithFilePath(filePath).WithContext("file_size", fileInfo.Size()).WithContext("size_limit", p.sizeLimit),
filePath,
context,
),
"Skipping large file %s", filePath,
)
return nil, fmt.Errorf("file too large")
}
// Check resource limits
if err := p.resourceMonitor.ValidateFileProcessing(filePath, fileInfo.Size()); err != nil {
utils.LogErrorf(err, "Resource limit validation failed for file: %s", filePath)
return nil, err
}
return fileInfo, nil
}
@@ -103,8 +192,20 @@ func (p *FileProcessor) getRelativePath(filePath string) string {
return relPath
}
// processInMemory loads the entire file into memory (for small files).
func (p *FileProcessor) processInMemory(filePath, relPath string, outCh chan<- WriteRequest) {
// processInMemoryWithContext loads the entire file into memory with context awareness.
func (p *FileProcessor) processInMemoryWithContext(ctx context.Context, filePath, relPath string, outCh chan<- WriteRequest) {
// Check context before reading
select {
case <-ctx.Done():
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled", filePath, nil),
"File processing cancelled: %s", filePath,
)
return
default:
}
content, err := os.ReadFile(filePath) // #nosec G304 - filePath is validated by walker
if err != nil {
structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to read file").WithFilePath(filePath)
@@ -112,30 +213,79 @@ func (p *FileProcessor) processInMemory(filePath, relPath string, outCh chan<- W
return
}
outCh <- WriteRequest{
// Check context again after reading
select {
case <-ctx.Done():
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled after read", filePath, nil),
"File processing cancelled after read: %s", filePath,
)
return
default:
}
// Try to send the result, but respect context cancellation
select {
case <-ctx.Done():
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled before output", filePath, nil),
"File processing cancelled before output: %s", filePath,
)
return
case outCh <- WriteRequest{
Path: relPath,
Content: p.formatContent(relPath, string(content)),
IsStream: false,
}:
}
}
// processStreaming creates a streaming reader for large files.
func (p *FileProcessor) processStreaming(filePath, relPath string, outCh chan<- WriteRequest) {
reader := p.createStreamReader(filePath, relPath)
// processStreamingWithContext creates a streaming reader for large files with context awareness.
func (p *FileProcessor) processStreamingWithContext(ctx context.Context, filePath, relPath string, outCh chan<- WriteRequest) {
// Check context before creating reader
select {
case <-ctx.Done():
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "streaming processing cancelled", filePath, nil),
"Streaming processing cancelled: %s", filePath,
)
return
default:
}
reader := p.createStreamReaderWithContext(ctx, filePath, relPath)
if reader == nil {
return // Error already logged
}
outCh <- WriteRequest{
// Try to send the result, but respect context cancellation
select {
case <-ctx.Done():
utils.LogErrorf(
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "streaming processing cancelled before output", filePath, nil),
"Streaming processing cancelled before output: %s", filePath,
)
return
case outCh <- WriteRequest{
Path: relPath,
Content: "", // Empty since content is in Reader
IsStream: true,
Reader: reader,
}:
}
}
// createStreamReader creates a reader that combines header and file content.
func (p *FileProcessor) createStreamReader(filePath, relPath string) io.Reader {
// createStreamReaderWithContext creates a reader that combines header and file content with context awareness.
func (p *FileProcessor) createStreamReaderWithContext(ctx context.Context, filePath, relPath string) io.Reader {
// Check context before opening file
select {
case <-ctx.Done():
return nil
default:
}
file, err := os.Open(filePath) // #nosec G304 - filePath is validated by walker
if err != nil {
structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to open file for streaming").WithFilePath(filePath)

View File

@@ -0,0 +1,423 @@
// Package fileproc provides resource monitoring and limit enforcement for security.
package fileproc
import (
"context"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/utils"
)
// ResourceMonitor monitors resource usage and enforces limits to prevent DoS attacks.
type ResourceMonitor struct {
enabled bool
maxFiles int
maxTotalSize int64
fileProcessingTimeout time.Duration
overallTimeout time.Duration
maxConcurrentReads int
rateLimitFilesPerSec int
hardMemoryLimitMB int
enableGracefulDegr bool
enableResourceMon bool
// Current state tracking
filesProcessed int64
totalSizeProcessed int64
concurrentReads int64
startTime time.Time
lastRateLimitCheck time.Time
hardMemoryLimitBytes int64
// Rate limiting
rateLimiter *time.Ticker
rateLimitChan chan struct{}
// Synchronization
mu sync.RWMutex
violationLogged map[string]bool
degradationActive bool
emergencyStopRequested bool
}
// ResourceMetrics holds comprehensive resource usage metrics.
type ResourceMetrics struct {
FilesProcessed int64 `json:"files_processed"`
TotalSizeProcessed int64 `json:"total_size_processed"`
ConcurrentReads int64 `json:"concurrent_reads"`
ProcessingDuration time.Duration `json:"processing_duration"`
AverageFileSize float64 `json:"average_file_size"`
ProcessingRate float64 `json:"processing_rate_files_per_sec"`
MemoryUsageMB int64 `json:"memory_usage_mb"`
MaxMemoryUsageMB int64 `json:"max_memory_usage_mb"`
ViolationsDetected []string `json:"violations_detected"`
DegradationActive bool `json:"degradation_active"`
EmergencyStopActive bool `json:"emergency_stop_active"`
LastUpdated time.Time `json:"last_updated"`
}
// ResourceViolation represents a detected resource limit violation.
type ResourceViolation struct {
Type string `json:"type"`
Message string `json:"message"`
Current interface{} `json:"current"`
Limit interface{} `json:"limit"`
Timestamp time.Time `json:"timestamp"`
Context map[string]interface{} `json:"context"`
}
// NewResourceMonitor creates a new resource monitor with configuration.
func NewResourceMonitor() *ResourceMonitor {
rm := &ResourceMonitor{
enabled: config.GetResourceLimitsEnabled(),
maxFiles: config.GetMaxFiles(),
maxTotalSize: config.GetMaxTotalSize(),
fileProcessingTimeout: time.Duration(config.GetFileProcessingTimeoutSec()) * time.Second,
overallTimeout: time.Duration(config.GetOverallTimeoutSec()) * time.Second,
maxConcurrentReads: config.GetMaxConcurrentReads(),
rateLimitFilesPerSec: config.GetRateLimitFilesPerSec(),
hardMemoryLimitMB: config.GetHardMemoryLimitMB(),
enableGracefulDegr: config.GetEnableGracefulDegradation(),
enableResourceMon: config.GetEnableResourceMonitoring(),
startTime: time.Now(),
lastRateLimitCheck: time.Now(),
violationLogged: make(map[string]bool),
hardMemoryLimitBytes: int64(config.GetHardMemoryLimitMB()) * 1024 * 1024,
}
// Initialize rate limiter if rate limiting is enabled
if rm.enabled && rm.rateLimitFilesPerSec > 0 {
interval := time.Second / time.Duration(rm.rateLimitFilesPerSec)
rm.rateLimiter = time.NewTicker(interval)
rm.rateLimitChan = make(chan struct{}, rm.rateLimitFilesPerSec)
// Pre-fill the rate limit channel
for i := 0; i < rm.rateLimitFilesPerSec; i++ {
select {
case rm.rateLimitChan <- struct{}{}:
default:
goto rateLimitFull
}
}
rateLimitFull:
// Start rate limiter refill goroutine
go rm.rateLimiterRefill()
}
return rm
}
// ValidateFileProcessing checks if a file can be processed based on resource limits.
func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int64) error {
if !rm.enabled {
return nil
}
rm.mu.RLock()
defer rm.mu.RUnlock()
// Check if emergency stop is active
if rm.emergencyStopRequested {
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitMemory,
"processing stopped due to emergency memory condition",
filePath,
map[string]interface{}{
"emergency_stop_active": true,
},
)
}
// Check file count limit
currentFiles := atomic.LoadInt64(&rm.filesProcessed)
if int(currentFiles) >= rm.maxFiles {
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitFiles,
"maximum file count limit exceeded",
filePath,
map[string]interface{}{
"current_files": currentFiles,
"max_files": rm.maxFiles,
},
)
}
// Check total size limit
currentTotalSize := atomic.LoadInt64(&rm.totalSizeProcessed)
if currentTotalSize+fileSize > rm.maxTotalSize {
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitTotalSize,
"maximum total size limit would be exceeded",
filePath,
map[string]interface{}{
"current_total_size": currentTotalSize,
"file_size": fileSize,
"max_total_size": rm.maxTotalSize,
},
)
}
// Check overall timeout
if time.Since(rm.startTime) > rm.overallTimeout {
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitTimeout,
"overall processing timeout exceeded",
filePath,
map[string]interface{}{
"processing_duration": time.Since(rm.startTime),
"overall_timeout": rm.overallTimeout,
},
)
}
return nil
}
// AcquireReadSlot attempts to acquire a slot for concurrent file reading.
func (rm *ResourceMonitor) AcquireReadSlot(ctx context.Context) error {
if !rm.enabled {
return nil
}
// Wait for available read slot
for {
currentReads := atomic.LoadInt64(&rm.concurrentReads)
if currentReads < int64(rm.maxConcurrentReads) {
if atomic.CompareAndSwapInt64(&rm.concurrentReads, currentReads, currentReads+1) {
break
}
// CAS failed, retry
continue
}
// Wait and retry
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Millisecond):
// Continue loop
}
}
return nil
}
// ReleaseReadSlot releases a concurrent reading slot.
func (rm *ResourceMonitor) ReleaseReadSlot() {
if rm.enabled {
atomic.AddInt64(&rm.concurrentReads, -1)
}
}
// WaitForRateLimit waits for rate limiting if enabled.
func (rm *ResourceMonitor) WaitForRateLimit(ctx context.Context) error {
if !rm.enabled || rm.rateLimitFilesPerSec <= 0 {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-rm.rateLimitChan:
return nil
case <-time.After(time.Second): // Fallback timeout
logrus.Warn("Rate limiting timeout exceeded, continuing without rate limit")
return nil
}
}
// CheckHardMemoryLimit checks if hard memory limit is exceeded and takes action.
func (rm *ResourceMonitor) CheckHardMemoryLimit() error {
if !rm.enabled || rm.hardMemoryLimitMB <= 0 {
return nil
}
var m runtime.MemStats
runtime.ReadMemStats(&m)
currentMemory := int64(m.Alloc)
if currentMemory > rm.hardMemoryLimitBytes {
rm.mu.Lock()
defer rm.mu.Unlock()
// Log violation if not already logged
violationKey := "hard_memory_limit"
if !rm.violationLogged[violationKey] {
logrus.Errorf("Hard memory limit exceeded: %dMB > %dMB",
currentMemory/1024/1024, rm.hardMemoryLimitMB)
rm.violationLogged[violationKey] = true
}
if rm.enableGracefulDegr {
// Force garbage collection
runtime.GC()
// Check again after GC
runtime.ReadMemStats(&m)
currentMemory = int64(m.Alloc)
if currentMemory > rm.hardMemoryLimitBytes {
// Still over limit, activate emergency stop
rm.emergencyStopRequested = true
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitMemory,
"hard memory limit exceeded, emergency stop activated",
"",
map[string]interface{}{
"current_memory_mb": currentMemory / 1024 / 1024,
"limit_mb": rm.hardMemoryLimitMB,
"emergency_stop": true,
},
)
} else {
// Memory freed by GC, continue with degradation
rm.degradationActive = true
logrus.Info("Memory freed by garbage collection, continuing with degradation mode")
}
} else {
// No graceful degradation, hard stop
return utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeResourceLimitMemory,
"hard memory limit exceeded",
"",
map[string]interface{}{
"current_memory_mb": currentMemory / 1024 / 1024,
"limit_mb": rm.hardMemoryLimitMB,
},
)
}
}
return nil
}
// RecordFileProcessed records that a file has been successfully processed.
func (rm *ResourceMonitor) RecordFileProcessed(fileSize int64) {
if rm.enabled {
atomic.AddInt64(&rm.filesProcessed, 1)
atomic.AddInt64(&rm.totalSizeProcessed, fileSize)
}
}
// GetMetrics returns current resource usage metrics.
func (rm *ResourceMonitor) GetMetrics() ResourceMetrics {
if !rm.enableResourceMon {
return ResourceMetrics{}
}
rm.mu.RLock()
defer rm.mu.RUnlock()
var m runtime.MemStats
runtime.ReadMemStats(&m)
filesProcessed := atomic.LoadInt64(&rm.filesProcessed)
totalSize := atomic.LoadInt64(&rm.totalSizeProcessed)
duration := time.Since(rm.startTime)
avgFileSize := float64(0)
if filesProcessed > 0 {
avgFileSize = float64(totalSize) / float64(filesProcessed)
}
processingRate := float64(0)
if duration.Seconds() > 0 {
processingRate = float64(filesProcessed) / duration.Seconds()
}
// Collect violations
violations := make([]string, 0, len(rm.violationLogged))
for violation := range rm.violationLogged {
violations = append(violations, violation)
}
return ResourceMetrics{
FilesProcessed: filesProcessed,
TotalSizeProcessed: totalSize,
ConcurrentReads: atomic.LoadInt64(&rm.concurrentReads),
ProcessingDuration: duration,
AverageFileSize: avgFileSize,
ProcessingRate: processingRate,
MemoryUsageMB: int64(m.Alloc) / 1024 / 1024,
MaxMemoryUsageMB: int64(rm.hardMemoryLimitMB),
ViolationsDetected: violations,
DegradationActive: rm.degradationActive,
EmergencyStopActive: rm.emergencyStopRequested,
LastUpdated: time.Now(),
}
}
// IsEmergencyStopActive returns whether emergency stop is active.
func (rm *ResourceMonitor) IsEmergencyStopActive() bool {
rm.mu.RLock()
defer rm.mu.RUnlock()
return rm.emergencyStopRequested
}
// IsDegradationActive returns whether degradation mode is active.
func (rm *ResourceMonitor) IsDegradationActive() bool {
rm.mu.RLock()
defer rm.mu.RUnlock()
return rm.degradationActive
}
// LogResourceInfo logs current resource limit configuration.
func (rm *ResourceMonitor) LogResourceInfo() {
if rm.enabled {
logrus.Infof("Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds",
rm.maxFiles, rm.maxTotalSize/1024/1024, int(rm.fileProcessingTimeout.Seconds()), int(rm.overallTimeout.Seconds()))
logrus.Infof("Resource limits: maxConcurrentReads=%d, rateLimitFPS=%d, hardMemoryMB=%d",
rm.maxConcurrentReads, rm.rateLimitFilesPerSec, rm.hardMemoryLimitMB)
logrus.Infof("Resource features: gracefulDegradation=%v, monitoring=%v",
rm.enableGracefulDegr, rm.enableResourceMon)
} else {
logrus.Info("Resource limits disabled")
}
}
// Close cleans up the resource monitor.
func (rm *ResourceMonitor) Close() {
if rm.rateLimiter != nil {
rm.rateLimiter.Stop()
}
}
// rateLimiterRefill refills the rate limiting channel periodically.
func (rm *ResourceMonitor) rateLimiterRefill() {
for range rm.rateLimiter.C {
select {
case rm.rateLimitChan <- struct{}{}:
default:
// Channel is full, skip
}
}
}
// CreateFileProcessingContext creates a context with file processing timeout.
func (rm *ResourceMonitor) CreateFileProcessingContext(parent context.Context) (context.Context, context.CancelFunc) {
if !rm.enabled || rm.fileProcessingTimeout <= 0 {
return parent, func() {}
}
return context.WithTimeout(parent, rm.fileProcessingTimeout)
}
// CreateOverallProcessingContext creates a context with overall processing timeout.
func (rm *ResourceMonitor) CreateOverallProcessingContext(parent context.Context) (context.Context, context.CancelFunc) {
if !rm.enabled || rm.overallTimeout <= 0 {
return parent, func() {}
}
return context.WithTimeout(parent, rm.overallTimeout)
}

View File

@@ -0,0 +1,377 @@
// Package fileproc provides tests for resource monitoring functionality.
package fileproc
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/spf13/viper"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/testutil"
"github.com/ivuorinen/gibidify/utils"
)
func TestResourceMonitor_NewResourceMonitor(t *testing.T) {
// Reset viper for clean test state
testutil.ResetViperConfig(t, "")
rm := NewResourceMonitor()
if rm == nil {
t.Fatal("NewResourceMonitor() returned nil")
}
// Test default values are set correctly
if !rm.enabled {
t.Error("Expected resource monitor to be enabled by default")
}
if rm.maxFiles != config.DefaultMaxFiles {
t.Errorf("Expected maxFiles to be %d, got %d", config.DefaultMaxFiles, rm.maxFiles)
}
if rm.maxTotalSize != config.DefaultMaxTotalSize {
t.Errorf("Expected maxTotalSize to be %d, got %d", config.DefaultMaxTotalSize, rm.maxTotalSize)
}
if rm.fileProcessingTimeout != time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second {
t.Errorf("Expected fileProcessingTimeout to be %v, got %v",
time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second, rm.fileProcessingTimeout)
}
// Clean up
rm.Close()
}
func TestResourceMonitor_DisabledResourceLimits(t *testing.T) {
// Reset viper for clean test state
testutil.ResetViperConfig(t, "")
// Set resource limits disabled
viper.Set("resourceLimits.enabled", false)
rm := NewResourceMonitor()
defer rm.Close()
// Test that validation passes when disabled
err := rm.ValidateFileProcessing("/tmp/test.txt", 1000)
if err != nil {
t.Errorf("Expected no error when resource limits disabled, got %v", err)
}
// Test that read slot acquisition works when disabled
ctx := context.Background()
err = rm.AcquireReadSlot(ctx)
if err != nil {
t.Errorf("Expected no error when acquiring read slot with disabled limits, got %v", err)
}
rm.ReleaseReadSlot()
// Test that rate limiting is bypassed when disabled
err = rm.WaitForRateLimit(ctx)
if err != nil {
t.Errorf("Expected no error when rate limiting disabled, got %v", err)
}
}
func TestResourceMonitor_FileCountLimit(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set a very low file count limit for testing
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.maxFiles", 2)
rm := NewResourceMonitor()
defer rm.Close()
// First file should pass
err := rm.ValidateFileProcessing("/tmp/file1.txt", 100)
if err != nil {
t.Errorf("Expected no error for first file, got %v", err)
}
rm.RecordFileProcessed(100)
// Second file should pass
err = rm.ValidateFileProcessing("/tmp/file2.txt", 100)
if err != nil {
t.Errorf("Expected no error for second file, got %v", err)
}
rm.RecordFileProcessed(100)
// Third file should fail
err = rm.ValidateFileProcessing("/tmp/file3.txt", 100)
if err == nil {
t.Error("Expected error for third file (exceeds limit), got nil")
}
// Verify it's the correct error type
structErr, ok := err.(*utils.StructuredError)
if !ok {
t.Errorf("Expected StructuredError, got %T", err)
} else if structErr.Code != utils.CodeResourceLimitFiles {
t.Errorf("Expected error code %s, got %s", utils.CodeResourceLimitFiles, structErr.Code)
}
}
func TestResourceMonitor_TotalSizeLimit(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set a low total size limit for testing (1KB)
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.maxTotalSize", 1024)
rm := NewResourceMonitor()
defer rm.Close()
// First small file should pass
err := rm.ValidateFileProcessing("/tmp/small.txt", 500)
if err != nil {
t.Errorf("Expected no error for small file, got %v", err)
}
rm.RecordFileProcessed(500)
// Second small file should pass
err = rm.ValidateFileProcessing("/tmp/small2.txt", 400)
if err != nil {
t.Errorf("Expected no error for second small file, got %v", err)
}
rm.RecordFileProcessed(400)
// Large file that would exceed limit should fail
err = rm.ValidateFileProcessing("/tmp/large.txt", 200)
if err == nil {
t.Error("Expected error for file that would exceed size limit, got nil")
}
// Verify it's the correct error type
structErr, ok := err.(*utils.StructuredError)
if !ok {
t.Errorf("Expected StructuredError, got %T", err)
} else if structErr.Code != utils.CodeResourceLimitTotalSize {
t.Errorf("Expected error code %s, got %s", utils.CodeResourceLimitTotalSize, structErr.Code)
}
}
func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set a low concurrent reads limit for testing
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.maxConcurrentReads", 2)
rm := NewResourceMonitor()
defer rm.Close()
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
// First read slot should succeed
err := rm.AcquireReadSlot(ctx)
if err != nil {
t.Errorf("Expected no error for first read slot, got %v", err)
}
// Second read slot should succeed
err = rm.AcquireReadSlot(ctx)
if err != nil {
t.Errorf("Expected no error for second read slot, got %v", err)
}
// Third read slot should timeout (context deadline exceeded)
err = rm.AcquireReadSlot(ctx)
if err == nil {
t.Error("Expected timeout error for third read slot, got nil")
}
// Release one slot and try again
rm.ReleaseReadSlot()
// Create new context for the next attempt
ctx2, cancel2 := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel2()
err = rm.AcquireReadSlot(ctx2)
if err != nil {
t.Errorf("Expected no error after releasing a slot, got %v", err)
}
// Clean up remaining slots
rm.ReleaseReadSlot()
rm.ReleaseReadSlot()
}
func TestResourceMonitor_TimeoutContexts(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set short timeouts for testing
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.fileProcessingTimeoutSec", 1) // 1 second
viper.Set("resourceLimits.overallTimeoutSec", 2) // 2 seconds
rm := NewResourceMonitor()
defer rm.Close()
parentCtx := context.Background()
// Test file processing context
fileCtx, fileCancel := rm.CreateFileProcessingContext(parentCtx)
defer fileCancel()
deadline, ok := fileCtx.Deadline()
if !ok {
t.Error("Expected file processing context to have a deadline")
} else if time.Until(deadline) > time.Second+100*time.Millisecond {
t.Error("File processing timeout appears to be too long")
}
// Test overall processing context
overallCtx, overallCancel := rm.CreateOverallProcessingContext(parentCtx)
defer overallCancel()
deadline, ok = overallCtx.Deadline()
if !ok {
t.Error("Expected overall processing context to have a deadline")
} else if time.Until(deadline) > 2*time.Second+100*time.Millisecond {
t.Error("Overall processing timeout appears to be too long")
}
}
func TestResourceMonitor_RateLimiting(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Enable rate limiting with a low rate for testing
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.rateLimitFilesPerSec", 5) // 5 files per second
rm := NewResourceMonitor()
defer rm.Close()
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
// First few requests should succeed quickly
start := time.Now()
for i := 0; i < 3; i++ {
err := rm.WaitForRateLimit(ctx)
if err != nil {
t.Errorf("Expected no error for rate limit wait %d, got %v", i, err)
}
}
// Should have taken some time due to rate limiting
duration := time.Since(start)
if duration < 200*time.Millisecond {
t.Logf("Rate limiting may not be working as expected, took only %v", duration)
}
}
func TestResourceMonitor_Metrics(t *testing.T) {
testutil.ResetViperConfig(t, "")
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.enableResourceMonitoring", true)
rm := NewResourceMonitor()
defer rm.Close()
// Process some files to generate metrics
rm.RecordFileProcessed(1000)
rm.RecordFileProcessed(2000)
rm.RecordFileProcessed(500)
metrics := rm.GetMetrics()
// Verify metrics
if metrics.FilesProcessed != 3 {
t.Errorf("Expected 3 files processed, got %d", metrics.FilesProcessed)
}
if metrics.TotalSizeProcessed != 3500 {
t.Errorf("Expected total size 3500, got %d", metrics.TotalSizeProcessed)
}
expectedAvgSize := float64(3500) / float64(3)
if metrics.AverageFileSize != expectedAvgSize {
t.Errorf("Expected average file size %.2f, got %.2f", expectedAvgSize, metrics.AverageFileSize)
}
if metrics.ProcessingRate <= 0 {
t.Error("Expected positive processing rate")
}
if !metrics.LastUpdated.After(time.Now().Add(-time.Second)) {
t.Error("Expected recent LastUpdated timestamp")
}
}
func TestResourceMonitor_Integration(t *testing.T) {
// Create temporary test directory
tempDir := t.TempDir()
// Create test files
testFiles := []string{"test1.txt", "test2.txt", "test3.txt"}
for _, filename := range testFiles {
testutil.CreateTestFile(t, tempDir, filename, []byte("test content"))
}
testutil.ResetViperConfig(t, "")
// Configure resource limits
viper.Set("resourceLimits.enabled", true)
viper.Set("resourceLimits.maxFiles", 5)
viper.Set("resourceLimits.maxTotalSize", 1024*1024) // 1MB
viper.Set("resourceLimits.fileProcessingTimeoutSec", 10)
viper.Set("resourceLimits.maxConcurrentReads", 3)
rm := NewResourceMonitor()
defer rm.Close()
ctx := context.Background()
// Test file processing workflow
for _, filename := range testFiles {
filePath := filepath.Join(tempDir, filename)
fileInfo, err := os.Stat(filePath)
if err != nil {
t.Fatalf("Failed to stat test file %s: %v", filePath, err)
}
// Validate file can be processed
err = rm.ValidateFileProcessing(filePath, fileInfo.Size())
if err != nil {
t.Errorf("Failed to validate file %s: %v", filePath, err)
continue
}
// Acquire read slot
err = rm.AcquireReadSlot(ctx)
if err != nil {
t.Errorf("Failed to acquire read slot for %s: %v", filePath, err)
continue
}
// Check memory limits
err = rm.CheckHardMemoryLimit()
if err != nil {
t.Errorf("Memory limit check failed for %s: %v", filePath, err)
}
// Record processing
rm.RecordFileProcessed(fileInfo.Size())
// Release read slot
rm.ReleaseReadSlot()
}
// Verify final metrics
metrics := rm.GetMetrics()
if metrics.FilesProcessed != int64(len(testFiles)) {
t.Errorf("Expected %d files processed, got %d", len(testFiles), metrics.FilesProcessed)
}
// Test resource limit logging
rm.LogResourceInfo()
}

View File

@@ -18,11 +18,16 @@ func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- stru
case "yaml":
startYAMLWriter(outFile, writeCh, done, prefix, suffix)
default:
context := map[string]interface{}{
"format": format,
}
err := utils.NewStructuredError(
utils.ErrorTypeValidation,
utils.CodeValidationFormat,
fmt.Sprintf("unsupported format: %s", format),
).WithContext("format", format)
"",
context,
)
utils.LogError("Failed to encode output", err)
close(done)
}

25
scripts/help.txt Normal file
View File

@@ -0,0 +1,25 @@
Available targets:
install-tools - Install required linting and development tools
lint - Run all linters (Go, Makefile, shell, YAML)
lint-fix - Run linters with auto-fix enabled
lint-verbose - Run linters with verbose output
test - Run tests
coverage - Run tests with coverage
build - Build the application
clean - Clean build artifacts
all - Run lint, test, and build
Security targets:
security - Run comprehensive security scan
security-full - Run full security analysis with all tools
vuln-check - Check for dependency vulnerabilities
Benchmark targets:
build-benchmark - Build the benchmark binary
benchmark - Run all benchmarks
benchmark-collection - Run file collection benchmarks
benchmark-processing - Run file processing benchmarks
benchmark-concurrency - Run concurrency benchmarks
benchmark-format - Run format benchmarks
Run 'make <target>' to execute a specific target.

14
scripts/lint.sh Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/bash
set -e
echo "Running golangci-lint..."
golangci-lint run ./...
echo "Running checkmake..."
checkmake --config=.checkmake Makefile
echo "Running shfmt check..."
shfmt -d .
echo "Running yamllint..."
yamllint -c .yamllint .

426
scripts/security-scan.sh Executable file
View File

@@ -0,0 +1,426 @@
#!/bin/bash
set -euo pipefail
# Security Scanning Script for gibidify
# This script runs comprehensive security checks locally and in CI
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_ROOT"
echo "🔒 Starting comprehensive security scan for gibidify..."
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Function to print status
print_status() {
echo -e "${BLUE}[INFO]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
print_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
# Check if required tools are installed
check_dependencies() {
print_status "Checking security scanning dependencies..."
local missing_tools=()
if ! command -v go &>/dev/null; then
missing_tools+=("go")
fi
if ! command -v golangci-lint &>/dev/null; then
print_warning "golangci-lint not found, installing..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
fi
if ! command -v gosec &>/dev/null; then
print_warning "gosec not found, installing..."
go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest
fi
if ! command -v govulncheck &>/dev/null; then
print_warning "govulncheck not found, installing..."
go install golang.org/x/vuln/cmd/govulncheck@latest
fi
if ! command -v checkmake &>/dev/null; then
print_warning "checkmake not found, installing..."
go install github.com/mrtazz/checkmake/cmd/checkmake@latest
fi
if ! command -v shfmt &>/dev/null; then
print_warning "shfmt not found, installing..."
go install mvdan.cc/sh/v3/cmd/shfmt@latest
fi
if ! command -v yamllint &>/dev/null; then
print_warning "yamllint not found, installing..."
go install github.com/excilsploft/yamllint@latest
fi
if [ ${#missing_tools[@]} -ne 0 ]; then
print_error "Missing required tools: ${missing_tools[*]}"
print_error "Please install the missing tools and try again."
exit 1
fi
print_success "All dependencies are available"
}
# Run gosec security scanner
run_gosec() {
print_status "Running gosec security scanner..."
if gosec -fmt=json -out=gosec-report.json -stdout -verbose=text ./...; then
print_success "gosec scan completed successfully"
else
print_error "gosec found security issues!"
if [ -f "gosec-report.json" ]; then
echo "Detailed report saved to gosec-report.json"
fi
return 1
fi
}
# Run vulnerability check
run_govulncheck() {
print_status "Running govulncheck for dependency vulnerabilities..."
if govulncheck -json ./... >govulncheck-report.json 2>&1; then
print_success "No known vulnerabilities found in dependencies"
else
if grep -q '"finding"' govulncheck-report.json 2>/dev/null; then
print_error "Vulnerabilities found in dependencies!"
echo "Detailed report saved to govulncheck-report.json"
return 1
else
print_success "No vulnerabilities found"
fi
fi
}
# Run enhanced golangci-lint with security focus
run_security_lint() {
print_status "Running security-focused linting..."
local security_linters="gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused,errcheck,ineffassign,staticcheck"
if golangci-lint run --enable="$security_linters" --timeout=5m; then
print_success "Security linting passed"
else
print_error "Security linting found issues!"
return 1
fi
}
# Check for potential secrets
check_secrets() {
print_status "Scanning for potential secrets and sensitive data..."
local secrets_found=false
# Common secret patterns
local patterns=(
"password\s*[:=]\s*['\"][^'\"]{3,}['\"]"
"secret\s*[:=]\s*['\"][^'\"]{3,}['\"]"
"key\s*[:=]\s*['\"][^'\"]{8,}['\"]"
"token\s*[:=]\s*['\"][^'\"]{8,}['\"]"
"api_?key\s*[:=]\s*['\"][^'\"]{8,}['\"]"
"aws_?access_?key"
"aws_?secret"
"AKIA[0-9A-Z]{16}" # AWS Access Key pattern
"github_?token"
"private_?key"
)
for pattern in "${patterns[@]}"; do
if grep -r -i -E "$pattern" --include="*.go" . 2>/dev/null; then
print_warning "Potential secret pattern found: $pattern"
secrets_found=true
fi
done
# Check git history for secrets (last 10 commits)
if git log --oneline -10 | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then
print_warning "Potential secrets mentioned in recent commit messages"
secrets_found=true
fi
if [ "$secrets_found" = true ]; then
print_warning "Potential secrets detected. Please review manually."
return 1
else
print_success "No obvious secrets detected"
fi
}
# Check for hardcoded network addresses
check_hardcoded_addresses() {
print_status "Checking for hardcoded network addresses..."
local addresses_found=false
# Look for IP addresses (excluding common safe ones)
if grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . |
grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" >/dev/null 2>&1; then
print_warning "Hardcoded IP addresses found:"
grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . |
grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" || true
addresses_found=true
fi
# Look for URLs (excluding documentation examples)
if grep -r -E "https?://[^/\s]+" --include="*.go" . |
grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{)" >/dev/null 2>&1; then
print_warning "Hardcoded URLs found:"
grep -r -E "https?://[^/\s]+" --include="*.go" . |
grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{)" || true
addresses_found=true
fi
if [ "$addresses_found" = true ]; then
print_warning "Hardcoded network addresses detected. Please review."
return 1
else
print_success "No hardcoded network addresses found"
fi
}
# Check Docker security (if Dockerfile exists)
check_docker_security() {
if [ -f "Dockerfile" ]; then
print_status "Checking Docker security..."
# Basic Dockerfile security checks
local docker_issues=false
if grep -q "^USER root" Dockerfile; then
print_warning "Dockerfile runs as root user"
docker_issues=true
fi
if ! grep -q "^USER " Dockerfile; then
print_warning "Dockerfile doesn't specify a non-root user"
docker_issues=true
fi
if grep -q "RUN.*wget\|RUN.*curl" Dockerfile && ! grep -q "rm.*wget\|rm.*curl" Dockerfile; then
print_warning "Dockerfile may leave curl/wget installed"
docker_issues=true
fi
if [ "$docker_issues" = true ]; then
print_warning "Docker security issues detected"
return 1
else
print_success "Docker security check passed"
fi
else
print_status "No Dockerfile found, skipping Docker security check"
fi
}
# Check file permissions
check_file_permissions() {
print_status "Checking file permissions..."
local perm_issues=false
# Check for overly permissive files
if find . -type f -perm /o+w -not -path "./.git/*" | grep -q .; then
print_warning "World-writable files found:"
find . -type f -perm /o+w -not -path "./.git/*" || true
perm_issues=true
fi
# Check for executable files that shouldn't be
if find . -type f -name "*.go" -perm /a+x | grep -q .; then
print_warning "Executable Go files found (should not be executable):"
find . -type f -name "*.go" -perm /a+x || true
perm_issues=true
fi
if [ "$perm_issues" = true ]; then
print_warning "File permission issues detected"
return 1
else
print_success "File permissions check passed"
fi
}
# Check Makefile with checkmake
check_makefile() {
if [ -f "Makefile" ]; then
print_status "Checking Makefile with checkmake..."
if checkmake --config=.checkmake Makefile; then
print_success "Makefile check passed"
else
print_error "Makefile issues detected!"
return 1
fi
else
print_status "No Makefile found, skipping checkmake"
fi
}
# Check shell scripts with shfmt
check_shell_scripts() {
print_status "Checking shell script formatting..."
if find . -name "*.sh" -type f | head -1 | grep -q .; then
if shfmt -d .; then
print_success "Shell script formatting check passed"
else
print_error "Shell script formatting issues detected!"
return 1
fi
else
print_status "No shell scripts found, skipping shfmt check"
fi
}
# Check YAML files
check_yaml_files() {
print_status "Checking YAML files..."
if find . -name "*.yml" -o -name "*.yaml" -type f | head -1 | grep -q .; then
if yamllint -c .yamllint .; then
print_success "YAML files check passed"
else
print_error "YAML file issues detected!"
return 1
fi
else
print_status "No YAML files found, skipping yamllint check"
fi
}
# Generate security report
generate_report() {
print_status "Generating security scan report..."
local report_file="security-report.md"
cat >"$report_file" <<EOF
# Security Scan Report
**Generated:** $(date)
**Project:** gibidify
**Scan Type:** Comprehensive Security Analysis
## Scan Results
### Security Tools Used
- gosec (Go security analyzer)
- govulncheck (Vulnerability database checker)
- golangci-lint (Static analysis with security linters)
- checkmake (Makefile linting)
- shfmt (Shell script formatting)
- yamllint (YAML file validation)
- Custom secret detection
- Custom network address detection
- Docker security checks
- File permission checks
### Files Generated
- \`gosec-report.json\` - Detailed gosec security findings
- \`govulncheck-report.json\` - Dependency vulnerability report
### Recommendations
1. Review all security findings in the generated reports
2. Address any HIGH or MEDIUM severity issues immediately
3. Consider implementing additional security measures for LOW severity issues
4. Regularly update dependencies to patch known vulnerabilities
5. Run security scans before each release
### Next Steps
- Fix any identified vulnerabilities
- Update security scanning in CI/CD pipeline
- Consider adding security testing to the test suite
- Review and update security documentation
---
*This report was generated automatically by the gibidify security scanning script.*
EOF
print_success "Security report generated: $report_file"
}
# Main execution
main() {
echo "🔒 gibidify Security Scanner"
echo "=========================="
echo
local exit_code=0
check_dependencies
echo
# Run all security checks
run_gosec || exit_code=1
echo
run_govulncheck || exit_code=1
echo
run_security_lint || exit_code=1
echo
check_secrets || exit_code=1
echo
check_hardcoded_addresses || exit_code=1
echo
check_docker_security || exit_code=1
echo
check_file_permissions || exit_code=1
echo
check_makefile || exit_code=1
echo
check_shell_scripts || exit_code=1
echo
check_yaml_files || exit_code=1
echo
generate_report
echo
if [ $exit_code -eq 0 ]; then
print_success "🎉 All security checks passed!"
else
print_error "❌ Security issues detected. Please review the reports and fix identified issues."
print_status "Generated reports:"
print_status "- gosec-report.json (if exists)"
print_status "- govulncheck-report.json (if exists)"
print_status "- security-report.md"
fi
exit $exit_code
}
# Run main function
main "$@"

View File

@@ -93,11 +93,13 @@ func (e *StructuredError) WithLine(line int) *StructuredError {
}
// NewStructuredError creates a new structured error.
func NewStructuredError(errorType ErrorType, code, message string) *StructuredError {
func NewStructuredError(errorType ErrorType, code, message, filePath string, context map[string]interface{}) *StructuredError {
return &StructuredError{
Type: errorType,
Code: code,
Message: message,
Type: errorType,
Code: code,
Message: message,
FilePath: filePath,
Context: context,
}
}
@@ -164,33 +166,43 @@ const (
CodeValidationFormat = "FORMAT"
CodeValidationFileType = "FILE_TYPE"
CodeValidationSize = "SIZE_LIMIT"
CodeValidationRequired = "REQUIRED"
CodeValidationPath = "PATH_TRAVERSAL"
// Resource Limit Error Codes
CodeResourceLimitFiles = "FILE_COUNT_LIMIT"
CodeResourceLimitTotalSize = "TOTAL_SIZE_LIMIT"
CodeResourceLimitTimeout = "TIMEOUT"
CodeResourceLimitMemory = "MEMORY_LIMIT"
CodeResourceLimitConcurrency = "CONCURRENCY_LIMIT"
CodeResourceLimitRate = "RATE_LIMIT"
)
// Predefined error constructors for common error scenarios
// NewCLIMissingSourceError creates a CLI error for missing source argument.
func NewCLIMissingSourceError() *StructuredError {
return NewStructuredError(ErrorTypeCLI, CodeCLIMissingSource, "usage: gibidify -source <source_directory> [--destination <output_file>] [--format=json|yaml|markdown]")
return NewStructuredError(ErrorTypeCLI, CodeCLIMissingSource, "usage: gibidify -source <source_directory> [--destination <output_file>] [--format=json|yaml|markdown]", "", nil)
}
// NewFileSystemError creates a file system error.
func NewFileSystemError(code, message string) *StructuredError {
return NewStructuredError(ErrorTypeFileSystem, code, message)
return NewStructuredError(ErrorTypeFileSystem, code, message, "", nil)
}
// NewProcessingError creates a processing error.
func NewProcessingError(code, message string) *StructuredError {
return NewStructuredError(ErrorTypeProcessing, code, message)
return NewStructuredError(ErrorTypeProcessing, code, message, "", nil)
}
// NewIOError creates an IO error.
func NewIOError(code, message string) *StructuredError {
return NewStructuredError(ErrorTypeIO, code, message)
return NewStructuredError(ErrorTypeIO, code, message, "", nil)
}
// NewValidationError creates a validation error.
func NewValidationError(code, message string) *StructuredError {
return NewStructuredError(ErrorTypeValidation, code, message)
return NewStructuredError(ErrorTypeValidation, code, message, "", nil)
}
// LogError logs an error with a consistent format if the error is not nil.

View File

@@ -3,7 +3,9 @@ package utils
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// GetAbsolutePath returns the absolute path for the given path.
@@ -24,3 +26,142 @@ func GetBaseName(absPath string) string {
}
return baseName
}
// ValidateSourcePath validates a source directory path for security.
// It ensures the path exists, is a directory, and doesn't contain path traversal attempts.
func ValidateSourcePath(path string) error {
if path == "" {
return NewStructuredError(ErrorTypeValidation, CodeValidationRequired, "source path is required", "", nil)
}
// Check for path traversal patterns before cleaning
if strings.Contains(path, "..") {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "path traversal attempt detected in source path", path, map[string]interface{}{
"original_path": path,
})
}
// Clean and get absolute path
cleaned := filepath.Clean(path)
abs, err := filepath.Abs(cleaned)
if err != nil {
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot resolve source path", path, map[string]interface{}{
"error": err.Error(),
})
}
// Get current working directory to ensure we're not escaping it for relative paths
if !filepath.IsAbs(path) {
cwd, err := os.Getwd()
if err != nil {
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot get current working directory", path, map[string]interface{}{
"error": err.Error(),
})
}
// Ensure the resolved path is within or below the current working directory
cwdAbs, err := filepath.Abs(cwd)
if err != nil {
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot resolve current working directory", path, map[string]interface{}{
"error": err.Error(),
})
}
// Check if the absolute path tries to escape the current working directory
if !strings.HasPrefix(abs, cwdAbs) {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "source path attempts to access directories outside current working directory", path, map[string]interface{}{
"resolved_path": abs,
"working_dir": cwdAbs,
})
}
}
// Check if path exists and is a directory
info, err := os.Stat(cleaned)
if err != nil {
if os.IsNotExist(err) {
return NewStructuredError(ErrorTypeFileSystem, CodeFSNotFound, "source directory does not exist", path, nil)
}
return NewStructuredError(ErrorTypeFileSystem, CodeFSAccess, "cannot access source directory", path, map[string]interface{}{
"error": err.Error(),
})
}
if !info.IsDir() {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "source path must be a directory", path, map[string]interface{}{
"is_file": true,
})
}
return nil
}
// ValidateDestinationPath validates a destination file path for security.
// It ensures the path doesn't contain path traversal attempts and the parent directory exists.
func ValidateDestinationPath(path string) error {
if path == "" {
return NewStructuredError(ErrorTypeValidation, CodeValidationRequired, "destination path is required", "", nil)
}
// Check for path traversal patterns before cleaning
if strings.Contains(path, "..") {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "path traversal attempt detected in destination path", path, map[string]interface{}{
"original_path": path,
})
}
// Clean and validate the path
cleaned := filepath.Clean(path)
// Get absolute path to ensure it's not trying to escape current working directory
abs, err := filepath.Abs(cleaned)
if err != nil {
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot resolve destination path", path, map[string]interface{}{
"error": err.Error(),
})
}
// Ensure the destination is not a directory
if info, err := os.Stat(abs); err == nil && info.IsDir() {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "destination cannot be a directory", path, map[string]interface{}{
"is_directory": true,
})
}
// Check if parent directory exists and is writable
parentDir := filepath.Dir(abs)
if parentInfo, err := os.Stat(parentDir); err != nil {
if os.IsNotExist(err) {
return NewStructuredError(ErrorTypeFileSystem, CodeFSNotFound, "destination parent directory does not exist", path, map[string]interface{}{
"parent_dir": parentDir,
})
}
return NewStructuredError(ErrorTypeFileSystem, CodeFSAccess, "cannot access destination parent directory", path, map[string]interface{}{
"parent_dir": parentDir,
"error": err.Error(),
})
} else if !parentInfo.IsDir() {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "destination parent is not a directory", path, map[string]interface{}{
"parent_dir": parentDir,
})
}
return nil
}
// ValidateConfigPath validates a configuration file path for security.
// It ensures the path doesn't contain path traversal attempts.
func ValidateConfigPath(path string) error {
if path == "" {
return nil // Empty path is allowed for config
}
// Check for path traversal patterns before cleaning
if strings.Contains(path, "..") {
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "path traversal attempt detected in config path", path, map[string]interface{}{
"original_path": path,
})
}
return nil
}