Files
gibidify/config/validation.go
Ismo Vuorinen 95b7ef6dd3 chore: modernize workflows, security scanning, and linting configuration (#50)
* build: update Go 1.25, CI workflows, and build tooling

- Upgrade to Go 1.25
- Add benchmark targets to Makefile
- Implement parallel gosec execution
- Lock tool versions for reproducibility
- Add shellcheck directives to scripts
- Update CI workflows with improved caching

* refactor: migrate from golangci-lint to revive

- Replace golangci-lint with revive for linting
- Configure comprehensive revive rules
- Fix all EditorConfig violations
- Add yamllint and yamlfmt support
- Remove deprecated .golangci.yml

* refactor: rename utils to shared and deduplicate code

- Rename utils package to shared
- Add shared constants package
- Deduplicate constants across packages
- Address CodeRabbit review feedback

* fix: resolve SonarQube issues and add safety guards

- Fix all 73 SonarQube OPEN issues
- Add nil guards for resourceMonitor, backpressure, metricsCollector
- Implement io.Closer for headerFileReader
- Propagate errors from processing helpers
- Add metrics and templates packages
- Improve error handling across codebase

* test: improve test infrastructure and coverage

- Add benchmarks for cli, fileproc, metrics
- Improve test coverage for cli, fileproc, config
- Refactor tests with helper functions
- Add shared test constants
- Fix test function naming conventions
- Reduce cognitive complexity in benchmark tests

* docs: update documentation and configuration examples

- Update CLAUDE.md with current project state
- Refresh README with new features
- Add usage and configuration examples
- Add SonarQube project configuration
- Consolidate config.example.yaml

* fix: resolve shellcheck warnings in scripts

- Use ./*.go instead of *.go to prevent dash-prefixed filenames
  from being interpreted as options (SC2035)
- Remove unreachable return statement after exit (SC2317)
- Remove obsolete gibidiutils/ directory reference

* chore(deps): upgrade go dependencies

* chore(lint): megalinter fixes

* fix: improve test coverage and fix file descriptor leaks

- Add defer r.Close() to fix pipe file descriptor leaks in benchmark tests
- Refactor TestProcessorConfigureFileTypes with helper functions and assertions
- Refactor TestProcessorLogFinalStats with output capture and keyword verification
- Use shared constants instead of literal strings (TestFilePNG, FormatMarkdown, etc.)
- Reduce cognitive complexity by extracting helper functions

* fix: align test comments with function names

Remove underscores from test comments to match actual function names:
- benchmark/benchmark_test.go (2 fixes)
- fileproc/filetypes_config_test.go (4 fixes)
- fileproc/filetypes_registry_test.go (6 fixes)
- fileproc/processor_test.go (6 fixes)
- fileproc/resource_monitor_types_test.go (4 fixes)
- fileproc/writer_test.go (3 fixes)

* fix: various test improvements and bug fixes

- Remove duplicate maxCacheSize check in filetypes_registry_test.go
- Shorten long comment in processor_test.go to stay under 120 chars
- Remove flaky time.Sleep in collector_test.go, use >= 0 assertion
- Close pipe reader in benchmark_test.go to fix file descriptor leak
- Use ContinueOnError in flags_test.go to match ResetFlags behavior
- Add nil check for p.ui in processor_workers.go before UpdateProgress
- Fix resource_monitor_validation_test.go by setting hardMemoryLimitBytes directly

* chore(yaml): add missing document start markers

Add --- document start to YAML files to satisfy yamllint:
- .github/workflows/codeql.yml
- .github/workflows/build-test-publish.yml
- .github/workflows/security.yml
- .github/actions/setup/action.yml

* fix: guard nil resourceMonitor and fix test deadlock

- Guard resourceMonitor before CreateFileProcessingContext call
- Add ui.UpdateProgress on emergency stop and path error returns
- Fix potential deadlock in TestProcessFile using wg.Go with defer close
2025-12-10 19:07:11 +02:00

621 lines
19 KiB
Go

// Package config handles application configuration management.
package config
import (
"fmt"
"strings"
"github.com/spf13/viper"
"github.com/ivuorinen/gibidify/shared"
)
// ValidateConfig validates the loaded configuration.
func ValidateConfig() error {
var validationErrors []string
// Validate basic settings
validationErrors = append(validationErrors, validateBasicSettings()...)
validationErrors = append(validationErrors, validateFileTypeSettings()...)
validationErrors = append(validationErrors, validateBackpressureSettings()...)
validationErrors = append(validationErrors, validateResourceLimitSettings()...)
if len(validationErrors) > 0 {
return shared.NewStructuredError(
shared.ErrorTypeConfiguration,
shared.CodeConfigValidation,
"configuration validation failed: "+strings.Join(validationErrors, "; "),
"",
map[string]any{"validation_errors": validationErrors},
)
}
return nil
}
// validateBasicSettings validates basic configuration settings.
func validateBasicSettings() []string {
var validationErrors []string
validationErrors = append(validationErrors, validateFileSizeLimit()...)
validationErrors = append(validationErrors, validateIgnoreDirectories()...)
validationErrors = append(validationErrors, validateSupportedFormats()...)
validationErrors = append(validationErrors, validateConcurrencySettings()...)
validationErrors = append(validationErrors, validateFilePatterns()...)
return validationErrors
}
// validateFileSizeLimit validates the file size limit setting.
func validateFileSizeLimit() []string {
var validationErrors []string
fileSizeLimit := viper.GetInt64(shared.ConfigKeyFileSizeLimit)
if fileSizeLimit < shared.ConfigFileSizeLimitMin {
validationErrors = append(
validationErrors,
fmt.Sprintf("fileSizeLimit (%d) is below minimum (%d)", fileSizeLimit, shared.ConfigFileSizeLimitMin),
)
}
if fileSizeLimit > shared.ConfigFileSizeLimitMax {
validationErrors = append(
validationErrors,
fmt.Sprintf("fileSizeLimit (%d) exceeds maximum (%d)", fileSizeLimit, shared.ConfigFileSizeLimitMax),
)
}
return validationErrors
}
// validateIgnoreDirectories validates the ignore directories setting.
func validateIgnoreDirectories() []string {
var validationErrors []string
ignoreDirectories := viper.GetStringSlice(shared.ConfigKeyIgnoreDirectories)
for i, dir := range ignoreDirectories {
if errMsg := validateEmptyElement(shared.ConfigKeyIgnoreDirectories, dir, i); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
continue
}
dir = strings.TrimSpace(dir)
if strings.Contains(dir, "/") {
validationErrors = append(
validationErrors,
fmt.Sprintf(
"ignoreDirectories[%d] (%s) contains path separator - only directory names are allowed", i, dir,
),
)
}
if strings.HasPrefix(dir, ".") && dir != ".git" && dir != ".vscode" && dir != ".idea" {
validationErrors = append(
validationErrors,
fmt.Sprintf("ignoreDirectories[%d] (%s) starts with dot - this may cause unexpected behavior", i, dir),
)
}
}
return validationErrors
}
// validateSupportedFormats validates the supported formats setting.
func validateSupportedFormats() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeySupportedFormats) {
return validationErrors
}
supportedFormats := viper.GetStringSlice(shared.ConfigKeySupportedFormats)
validFormats := map[string]bool{shared.FormatJSON: true, shared.FormatYAML: true, shared.FormatMarkdown: true}
for i, format := range supportedFormats {
format = strings.ToLower(strings.TrimSpace(format))
if !validFormats[format] {
validationErrors = append(
validationErrors,
fmt.Sprintf("supportedFormats[%d] (%s) is not a valid format (json, yaml, markdown)", i, format),
)
}
}
return validationErrors
}
// validateConcurrencySettings validates the concurrency settings.
func validateConcurrencySettings() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyMaxConcurrency) {
return validationErrors
}
maxConcurrency := viper.GetInt(shared.ConfigKeyMaxConcurrency)
if maxConcurrency < 1 {
validationErrors = append(
validationErrors, fmt.Sprintf("maxConcurrency (%d) must be at least 1", maxConcurrency),
)
}
if maxConcurrency > 100 {
validationErrors = append(
validationErrors,
fmt.Sprintf("maxConcurrency (%d) is unreasonably high (max 100)", maxConcurrency),
)
}
return validationErrors
}
// validateFilePatterns validates the file patterns setting.
func validateFilePatterns() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyFilePatterns) {
return validationErrors
}
filePatterns := viper.GetStringSlice(shared.ConfigKeyFilePatterns)
for i, pattern := range filePatterns {
if errMsg := validateEmptyElement(shared.ConfigKeyFilePatterns, pattern, i); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
continue
}
pattern = strings.TrimSpace(pattern)
// Basic validation - patterns should contain at least one alphanumeric character
if !strings.ContainsAny(pattern, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") {
validationErrors = append(
validationErrors,
fmt.Sprintf("filePatterns[%d] (%s) appears to be invalid", i, pattern),
)
}
}
return validationErrors
}
// validateFileTypeSettings validates file type configuration settings.
func validateFileTypeSettings() []string {
var validationErrors []string
validationErrors = append(validationErrors, validateCustomImageExtensions()...)
validationErrors = append(validationErrors, validateCustomBinaryExtensions()...)
validationErrors = append(validationErrors, validateCustomLanguages()...)
return validationErrors
}
// validateCustomImageExtensions validates custom image extensions.
func validateCustomImageExtensions() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyFileTypesCustomImageExtensions) {
return validationErrors
}
customImages := viper.GetStringSlice(shared.ConfigKeyFileTypesCustomImageExtensions)
for i, ext := range customImages {
if errMsg := validateEmptyElement(shared.ConfigKeyFileTypesCustomImageExtensions, ext, i); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
continue
}
ext = strings.TrimSpace(ext)
if errMsg := validateDotPrefix(shared.ConfigKeyFileTypesCustomImageExtensions, ext, i); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
}
}
return validationErrors
}
// validateCustomBinaryExtensions validates custom binary extensions.
func validateCustomBinaryExtensions() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyFileTypesCustomBinaryExtensions) {
return validationErrors
}
customBinary := viper.GetStringSlice(shared.ConfigKeyFileTypesCustomBinaryExtensions)
for i, ext := range customBinary {
if errMsg := validateEmptyElement(shared.ConfigKeyFileTypesCustomBinaryExtensions, ext, i); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
continue
}
ext = strings.TrimSpace(ext)
if errMsg := validateDotPrefix(shared.ConfigKeyFileTypesCustomBinaryExtensions, ext, i); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
}
}
return validationErrors
}
// validateCustomLanguages validates custom language mappings.
func validateCustomLanguages() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyFileTypesCustomLanguages) {
return validationErrors
}
customLangs := viper.GetStringMapString(shared.ConfigKeyFileTypesCustomLanguages)
for ext, lang := range customLangs {
ext = strings.TrimSpace(ext)
if ext == "" {
validationErrors = append(
validationErrors,
shared.ConfigKeyFileTypesCustomLanguages+" contains empty extension key",
)
continue
}
if errMsg := validateDotPrefixMap(shared.ConfigKeyFileTypesCustomLanguages, ext); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
}
if errMsg := validateEmptyMapValue(shared.ConfigKeyFileTypesCustomLanguages, ext, lang); errMsg != "" {
validationErrors = append(validationErrors, errMsg)
}
}
return validationErrors
}
// validateBackpressureSettings validates back-pressure configuration settings.
func validateBackpressureSettings() []string {
var validationErrors []string
validationErrors = append(validationErrors, validateMaxPendingFiles()...)
validationErrors = append(validationErrors, validateMaxPendingWrites()...)
validationErrors = append(validationErrors, validateMaxMemoryUsage()...)
validationErrors = append(validationErrors, validateMemoryCheckInterval()...)
return validationErrors
}
// validateMaxPendingFiles validates backpressure.maxPendingFiles setting.
func validateMaxPendingFiles() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyBackpressureMaxPendingFiles) {
return validationErrors
}
maxPendingFiles := viper.GetInt(shared.ConfigKeyBackpressureMaxPendingFiles)
if maxPendingFiles < 1 {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.maxPendingFiles (%d) must be at least 1", maxPendingFiles),
)
}
if maxPendingFiles > 100000 {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.maxPendingFiles (%d) is unreasonably high (max 100000)", maxPendingFiles),
)
}
return validationErrors
}
// validateMaxPendingWrites validates backpressure.maxPendingWrites setting.
func validateMaxPendingWrites() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyBackpressureMaxPendingWrites) {
return validationErrors
}
maxPendingWrites := viper.GetInt(shared.ConfigKeyBackpressureMaxPendingWrites)
if maxPendingWrites < 1 {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.maxPendingWrites (%d) must be at least 1", maxPendingWrites),
)
}
if maxPendingWrites > 10000 {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.maxPendingWrites (%d) is unreasonably high (max 10000)", maxPendingWrites),
)
}
return validationErrors
}
// validateMaxMemoryUsage validates backpressure.maxMemoryUsage setting.
func validateMaxMemoryUsage() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyBackpressureMaxMemoryUsage) {
return validationErrors
}
maxMemoryUsage := viper.GetInt64(shared.ConfigKeyBackpressureMaxMemoryUsage)
minMemory := int64(shared.BytesPerMB) // 1MB minimum
maxMemory := int64(10 * shared.BytesPerGB) // 10GB maximum
if maxMemoryUsage < minMemory {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.maxMemoryUsage (%d) must be at least 1MB (%d bytes)", maxMemoryUsage, minMemory),
)
}
if maxMemoryUsage > maxMemory { // 10GB maximum
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.maxMemoryUsage (%d) is unreasonably high (max 10GB)", maxMemoryUsage),
)
}
return validationErrors
}
// validateMemoryCheckInterval validates backpressure.memoryCheckInterval setting.
func validateMemoryCheckInterval() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyBackpressureMemoryCheckInt) {
return validationErrors
}
interval := viper.GetInt(shared.ConfigKeyBackpressureMemoryCheckInt)
if interval < 1 {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.memoryCheckInterval (%d) must be at least 1", interval),
)
}
if interval > 100000 {
validationErrors = append(
validationErrors,
fmt.Sprintf("backpressure.memoryCheckInterval (%d) is unreasonably high (max 100000)", interval),
)
}
return validationErrors
}
// validateResourceLimitSettings validates resource limit configuration settings.
func validateResourceLimitSettings() []string {
var validationErrors []string
validationErrors = append(validationErrors, validateMaxFilesLimit()...)
validationErrors = append(validationErrors, validateMaxTotalSizeLimit()...)
validationErrors = append(validationErrors, validateTimeoutLimits()...)
validationErrors = append(validationErrors, validateConcurrencyLimits()...)
validationErrors = append(validationErrors, validateMemoryLimits()...)
return validationErrors
}
// validateMaxFilesLimit validates resourceLimits.maxFiles setting.
func validateMaxFilesLimit() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyResourceLimitsMaxFiles) {
return validationErrors
}
maxFiles := viper.GetInt(shared.ConfigKeyResourceLimitsMaxFiles)
if maxFiles < shared.ConfigMaxFilesMin {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.maxFiles (%d) must be at least %d", maxFiles, shared.ConfigMaxFilesMin),
)
}
if maxFiles > shared.ConfigMaxFilesMax {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.maxFiles (%d) exceeds maximum (%d)", maxFiles, shared.ConfigMaxFilesMax),
)
}
return validationErrors
}
// validateMaxTotalSizeLimit validates resourceLimits.maxTotalSize setting.
func validateMaxTotalSizeLimit() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyResourceLimitsMaxTotalSize) {
return validationErrors
}
maxTotalSize := viper.GetInt64(shared.ConfigKeyResourceLimitsMaxTotalSize)
minTotalSize := int64(shared.ConfigMaxTotalSizeMin)
maxTotalSizeLimit := int64(shared.ConfigMaxTotalSizeMax)
if maxTotalSize < minTotalSize {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.maxTotalSize (%d) must be at least %d", maxTotalSize, minTotalSize),
)
}
if maxTotalSize > maxTotalSizeLimit {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.maxTotalSize (%d) exceeds maximum (%d)", maxTotalSize, maxTotalSizeLimit),
)
}
return validationErrors
}
// validateTimeoutLimits validates timeout-related resource limit settings.
func validateTimeoutLimits() []string {
var validationErrors []string
if viper.IsSet(shared.ConfigKeyResourceLimitsFileProcessingTO) {
timeout := viper.GetInt(shared.ConfigKeyResourceLimitsFileProcessingTO)
if timeout < shared.ConfigFileProcessingTimeoutSecMin {
validationErrors = append(
validationErrors,
fmt.Sprintf(
"resourceLimits.fileProcessingTimeoutSec (%d) must be at least %d",
timeout,
shared.ConfigFileProcessingTimeoutSecMin,
),
)
}
if timeout > shared.ConfigFileProcessingTimeoutSecMax {
validationErrors = append(
validationErrors,
fmt.Sprintf(
"resourceLimits.fileProcessingTimeoutSec (%d) exceeds maximum (%d)",
timeout,
shared.ConfigFileProcessingTimeoutSecMax,
),
)
}
}
if viper.IsSet(shared.ConfigKeyResourceLimitsOverallTO) {
timeout := viper.GetInt(shared.ConfigKeyResourceLimitsOverallTO)
minTimeout := shared.ConfigOverallTimeoutSecMin
maxTimeout := shared.ConfigOverallTimeoutSecMax
if timeout < minTimeout {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) must be at least %d", timeout, minTimeout),
)
}
if timeout > maxTimeout {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) exceeds maximum (%d)", timeout, maxTimeout),
)
}
}
return validationErrors
}
// validateConcurrencyLimits validates concurrency-related resource limit settings.
func validateConcurrencyLimits() []string {
var validationErrors []string
if viper.IsSet(shared.ConfigKeyResourceLimitsMaxConcurrentReads) {
maxReads := viper.GetInt(shared.ConfigKeyResourceLimitsMaxConcurrentReads)
minReads := shared.ConfigMaxConcurrentReadsMin
maxReadsLimit := shared.ConfigMaxConcurrentReadsMax
if maxReads < minReads {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) must be at least %d", maxReads, minReads),
)
}
if maxReads > maxReadsLimit {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) exceeds maximum (%d)", maxReads, maxReadsLimit),
)
}
}
if viper.IsSet(shared.ConfigKeyResourceLimitsRateLimitFilesPerSec) {
rateLimit := viper.GetInt(shared.ConfigKeyResourceLimitsRateLimitFilesPerSec)
minRate := shared.ConfigRateLimitFilesPerSecMin
maxRate := shared.ConfigRateLimitFilesPerSecMax
if rateLimit < minRate {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) must be at least %d", rateLimit, minRate),
)
}
if rateLimit > maxRate {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) exceeds maximum (%d)", rateLimit, maxRate),
)
}
}
return validationErrors
}
// validateMemoryLimits validates memory-related resource limit settings.
func validateMemoryLimits() []string {
var validationErrors []string
if !viper.IsSet(shared.ConfigKeyResourceLimitsHardMemoryLimitMB) {
return validationErrors
}
memLimit := viper.GetInt(shared.ConfigKeyResourceLimitsHardMemoryLimitMB)
minMemLimit := shared.ConfigHardMemoryLimitMBMin
maxMemLimit := shared.ConfigHardMemoryLimitMBMax
if memLimit < minMemLimit {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) must be at least %d", memLimit, minMemLimit),
)
}
if memLimit > maxMemLimit {
validationErrors = append(
validationErrors,
fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) exceeds maximum (%d)", memLimit, maxMemLimit),
)
}
return validationErrors
}
// ValidateFileSize checks if a file size is within the configured limit.
func ValidateFileSize(size int64) error {
limit := FileSizeLimit()
if size > limit {
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeValidationSize,
fmt.Sprintf(shared.FileProcessingMsgSizeExceeds, size, limit),
"",
map[string]any{"file_size": size, "size_limit": limit},
)
}
return nil
}
// ValidateOutputFormat checks if an output format is valid.
func ValidateOutputFormat(format string) error {
if !IsValidFormat(format) {
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeValidationFormat,
fmt.Sprintf("unsupported output format: %s (supported: json, yaml, markdown)", format),
"",
map[string]any{"format": format},
)
}
return nil
}
// ValidateConcurrency checks if a concurrency level is valid.
func ValidateConcurrency(concurrency int) error {
if concurrency < 1 {
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeValidationFormat,
fmt.Sprintf("concurrency (%d) must be at least 1", concurrency),
"",
map[string]any{"concurrency": concurrency},
)
}
if viper.IsSet(shared.ConfigKeyMaxConcurrency) {
maxConcurrency := MaxConcurrency()
if concurrency > maxConcurrency {
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeValidationFormat,
fmt.Sprintf("concurrency (%d) exceeds maximum (%d)", concurrency, maxConcurrency),
"",
map[string]any{"concurrency": concurrency, "max_concurrency": maxConcurrency},
)
}
}
return nil
}