ivuorinen/gibidify
1
0
Fork 0
mirror of https://github.com/ivuorinen/gibidify.git synced 2026-02-12 08:48:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: modernize workflows, security scanning, and linting configuration (#50)

Browse Source 
* build: update Go 1.25, CI workflows, and build tooling

- Upgrade to Go 1.25
- Add benchmark targets to Makefile
- Implement parallel gosec execution
- Lock tool versions for reproducibility
- Add shellcheck directives to scripts
- Update CI workflows with improved caching

* refactor: migrate from golangci-lint to revive

- Replace golangci-lint with revive for linting
- Configure comprehensive revive rules
- Fix all EditorConfig violations
- Add yamllint and yamlfmt support
- Remove deprecated .golangci.yml

* refactor: rename utils to shared and deduplicate code

- Rename utils package to shared
- Add shared constants package
- Deduplicate constants across packages
- Address CodeRabbit review feedback

* fix: resolve SonarQube issues and add safety guards

- Fix all 73 SonarQube OPEN issues
- Add nil guards for resourceMonitor, backpressure, metricsCollector
- Implement io.Closer for headerFileReader
- Propagate errors from processing helpers
- Add metrics and templates packages
- Improve error handling across codebase

* test: improve test infrastructure and coverage

- Add benchmarks for cli, fileproc, metrics
- Improve test coverage for cli, fileproc, config
- Refactor tests with helper functions
- Add shared test constants
- Fix test function naming conventions
- Reduce cognitive complexity in benchmark tests

* docs: update documentation and configuration examples

- Update CLAUDE.md with current project state
- Refresh README with new features
- Add usage and configuration examples
- Add SonarQube project configuration
- Consolidate config.example.yaml

* fix: resolve shellcheck warnings in scripts

- Use ./*.go instead of *.go to prevent dash-prefixed filenames
  from being interpreted as options (SC2035)
- Remove unreachable return statement after exit (SC2317)
- Remove obsolete gibidiutils/ directory reference

* chore(deps): upgrade go dependencies

* chore(lint): megalinter fixes

* fix: improve test coverage and fix file descriptor leaks

- Add defer r.Close() to fix pipe file descriptor leaks in benchmark tests
- Refactor TestProcessorConfigureFileTypes with helper functions and assertions
- Refactor TestProcessorLogFinalStats with output capture and keyword verification
- Use shared constants instead of literal strings (TestFilePNG, FormatMarkdown, etc.)
- Reduce cognitive complexity by extracting helper functions

* fix: align test comments with function names

Remove underscores from test comments to match actual function names:
- benchmark/benchmark_test.go (2 fixes)
- fileproc/filetypes_config_test.go (4 fixes)
- fileproc/filetypes_registry_test.go (6 fixes)
- fileproc/processor_test.go (6 fixes)
- fileproc/resource_monitor_types_test.go (4 fixes)
- fileproc/writer_test.go (3 fixes)

* fix: various test improvements and bug fixes

- Remove duplicate maxCacheSize check in filetypes_registry_test.go
- Shorten long comment in processor_test.go to stay under 120 chars
- Remove flaky time.Sleep in collector_test.go, use >= 0 assertion
- Close pipe reader in benchmark_test.go to fix file descriptor leak
- Use ContinueOnError in flags_test.go to match ResetFlags behavior
- Add nil check for p.ui in processor_workers.go before UpdateProgress
- Fix resource_monitor_validation_test.go by setting hardMemoryLimitBytes directly

* chore(yaml): add missing document start markers

Add --- document start to YAML files to satisfy yamllint:
- .github/workflows/codeql.yml
- .github/workflows/build-test-publish.yml
- .github/workflows/security.yml
- .github/actions/setup/action.yml

* fix: guard nil resourceMonitor and fix test deadlock

- Guard resourceMonitor before CreateFileProcessingContext call
- Add ui.UpdateProgress on emergency stop and path error returns
- Fix potential deadlock in TestProcessFile using wg.Go with defer close
This commit is contained in:
Ismo Vuorinen
2025-12-10 19:07:11 +02:00
committed by GitHub
parent ea4a39a360
commit 95b7ef6dd3
149 changed files with 22990 additions and 8976 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
fileproc/backpressure.go
View File

@@ -3,16 +3,13 @@ package fileproc
import (
"context"
"math"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// BackpressureManager manages memory usage and applies back-pressure when needed.
@@ -31,11 +28,11 @@ type BackpressureManager struct {
// NewBackpressureManager creates a new back-pressure manager with configuration.
func NewBackpressureManager() *BackpressureManager {
return &BackpressureManager{
enabled: config.GetBackpressureEnabled(),
maxMemoryUsage: config.GetMaxMemoryUsage(),
memoryCheckInterval: config.GetMemoryCheckInterval(),
maxPendingFiles: config.GetMaxPendingFiles(),
maxPendingWrites: config.GetMaxPendingWrites(),
enabled: config.BackpressureEnabled(),
maxMemoryUsage: config.MaxMemoryUsage(),
memoryCheckInterval: config.MemoryCheckInterval(),
maxPendingFiles: config.MaxPendingFiles(),
maxPendingWrites: config.MaxPendingWrites(),
lastMemoryCheck: time.Now(),
}
}
@@ -45,38 +42,52 @@ func (bp *BackpressureManager) CreateChannels() (chan string, chan WriteRequest)
var fileCh chan string
var writeCh chan WriteRequest
logger := shared.GetLogger()
if bp.enabled {
// Use buffered channels with configured limits
fileCh = make(chan string, bp.maxPendingFiles)
writeCh = make(chan WriteRequest, bp.maxPendingWrites)
logrus.Debugf("Created buffered channels: files=%d, writes=%d", bp.maxPendingFiles, bp.maxPendingWrites)
logger.Debugf("Created buffered channels: files=%d, writes=%d", bp.maxPendingFiles, bp.maxPendingWrites)
} else {
// Use unbuffered channels (default behavior)
fileCh = make(chan string)
writeCh = make(chan WriteRequest)
logrus.Debug("Created unbuffered channels (back-pressure disabled)")
logger.Debug("Created unbuffered channels (back-pressure disabled)")
}
return fileCh, writeCh
}
// ShouldApplyBackpressure checks if back-pressure should be applied.
func (bp *BackpressureManager) ShouldApplyBackpressure(_ context.Context) bool {
func (bp *BackpressureManager) ShouldApplyBackpressure(ctx context.Context) bool {
// Check for context cancellation first
select {
case <-ctx.Done():
return false // No need for backpressure if canceled
default:
}
if !bp.enabled {
return false
}
// Check if we should evaluate memory usage
filesProcessed := atomic.AddInt64(&bp.filesProcessed, 1)
// Avoid divide by zero - if interval is 0, check every file
if bp.memoryCheckInterval > 0 && int(filesProcessed)%bp.memoryCheckInterval != 0 {
// Guard against zero or negative interval to avoid modulo-by-zero panic
interval := bp.memoryCheckInterval
if interval <= 0 {
interval = 1
}
if int(filesProcessed)%interval != 0 {
return false
}
// Get current memory usage
var m runtime.MemStats
runtime.ReadMemStats(&m)
currentMemory := gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, math.MaxInt64)
currentMemory := shared.SafeUint64ToInt64WithDefault(m.Alloc, 0)
bp.mu.Lock()
defer bp.mu.Unlock()
@@ -84,18 +95,22 @@ func (bp *BackpressureManager) ShouldApplyBackpressure(_ context.Context) bool {
bp.lastMemoryCheck = time.Now()
// Check if we're over the memory limit
logger := shared.GetLogger()
if currentMemory > bp.maxMemoryUsage {
if !bp.memoryWarningLogged {
logrus.Warnf("Memory usage (%d bytes) exceeds limit (%d bytes), applying back-pressure",
currentMemory, bp.maxMemoryUsage)
logger.Warnf(
"Memory usage (%d bytes) exceeds limit (%d bytes), applying back-pressure",
currentMemory, bp.maxMemoryUsage,
)
bp.memoryWarningLogged = true
}
return true
}
// Reset warning flag if we're back under the limit
if bp.memoryWarningLogged && currentMemory < bp.maxMemoryUsage*8/10 { // 80% of limit
logrus.Infof("Memory usage normalized (%d bytes), removing back-pressure", currentMemory)
logger.Infof("Memory usage normalized (%d bytes), removing back-pressure", currentMemory)
bp.memoryWarningLogged = false
}
@@ -108,14 +123,6 @@ func (bp *BackpressureManager) ApplyBackpressure(ctx context.Context) {
return
}
// Check for context cancellation before doing expensive operations
select {
case <-ctx.Done():
return
default:
// Continue with backpressure logic
}
// Force garbage collection to free up memory
runtime.GC()
@@ -130,11 +137,12 @@ func (bp *BackpressureManager) ApplyBackpressure(ctx context.Context) {
// Log memory usage after GC
var m runtime.MemStats
runtime.ReadMemStats(&m)
logrus.Debugf("Applied back-pressure: memory after GC = %d bytes", m.Alloc)
logger := shared.GetLogger()
logger.Debugf("Applied back-pressure: memory after GC = %d bytes", m.Alloc)
}
// GetStats returns current back-pressure statistics.
func (bp *BackpressureManager) GetStats() BackpressureStats {
// Stats returns current back-pressure statistics.
func (bp *BackpressureManager) Stats() BackpressureStats {
bp.mu.RLock()
defer bp.mu.RUnlock()
@@ -144,7 +152,7 @@ func (bp *BackpressureManager) GetStats() BackpressureStats {
return BackpressureStats{
Enabled: bp.enabled,
FilesProcessed: atomic.LoadInt64(&bp.filesProcessed),
CurrentMemoryUsage: gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, math.MaxInt64),
CurrentMemoryUsage: shared.SafeUint64ToInt64WithDefault(m.Alloc, 0),
MaxMemoryUsage: bp.maxMemoryUsage,
MemoryWarningActive: bp.memoryWarningLogged,
LastMemoryCheck: bp.lastMemoryCheck,
@@ -171,9 +179,11 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c
return
}
// Check if file channel is getting full (>=90% capacity)
if bp.maxPendingFiles > 0 && len(fileCh) >= bp.maxPendingFiles*9/10 {
logrus.Debugf("File channel is %d%% full, waiting for space", len(fileCh)*100/bp.maxPendingFiles)
logger := shared.GetLogger()
// Check if file channel is getting full (>90% capacity)
fileCap := cap(fileCh)
if fileCap > 0 && len(fileCh) > fileCap*9/10 {
logger.Debugf("File channel is %d%% full, waiting for space", len(fileCh)*100/fileCap)
// Wait a bit for the channel to drain
select {
@@ -183,9 +193,10 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c
}
}
// Check if write channel is getting full (>=90% capacity)
if bp.maxPendingWrites > 0 && len(writeCh) >= bp.maxPendingWrites*9/10 {
logrus.Debugf("Write channel is %d%% full, waiting for space", len(writeCh)*100/bp.maxPendingWrites)
// Check if write channel is getting full (>90% capacity)
writeCap := cap(writeCh)
if writeCap > 0 && len(writeCh) > writeCap*9/10 {
logger.Debugf("Write channel is %d%% full, waiting for space", len(writeCh)*100/writeCap)
// Wait a bit for the channel to drain
select {
@@ -198,10 +209,13 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c
// LogBackpressureInfo logs back-pressure configuration and status.
func (bp *BackpressureManager) LogBackpressureInfo() {
logger := shared.GetLogger()
if bp.enabled {
logrus.Infof("Back-pressure enabled: maxMemory=%dMB, fileBuffer=%d, writeBuffer=%d, checkInterval=%d",
bp.maxMemoryUsage/1024/1024, bp.maxPendingFiles, bp.maxPendingWrites, bp.memoryCheckInterval)
logger.Infof(
"Back-pressure enabled: maxMemory=%dMB, fileBuffer=%d, writeBuffer=%d, checkInterval=%d",
bp.maxMemoryUsage/int64(shared.BytesPerMB), bp.maxPendingFiles, bp.maxPendingWrites, bp.memoryCheckInterval,
)
} else {
logrus.Info("Back-pressure disabled")
logger.Info("Back-pressure disabled")
}
}

177
fileproc/backpressure_behavior_test.go
View File

@@ -1,177 +0,0 @@
package fileproc
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestBackpressureManagerShouldApplyBackpressure(t *testing.T) {
ctx := context.Background()
t.Run("returns false when disabled", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = false
shouldApply := bm.ShouldApplyBackpressure(ctx)
assert.False(t, shouldApply)
})
t.Run("checks memory at intervals", func(_ *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true
bm.memoryCheckInterval = 10
// Should not check memory on most calls
for i := 1; i < 10; i++ {
shouldApply := bm.ShouldApplyBackpressure(ctx)
// Can't predict result, but shouldn't panic
_ = shouldApply
}
// Should check memory on 10th call
shouldApply := bm.ShouldApplyBackpressure(ctx)
// Result depends on actual memory usage
_ = shouldApply
})
t.Run("detects high memory usage", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true
bm.memoryCheckInterval = 1
bm.maxMemoryUsage = 1 // Set very low limit to trigger
shouldApply := bm.ShouldApplyBackpressure(ctx)
// Should detect high memory usage
assert.True(t, shouldApply)
})
}
func TestBackpressureManagerApplyBackpressure(t *testing.T) {
ctx := context.Background()
t.Run("does nothing when disabled", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = false
// Use a channel to verify the function returns quickly
done := make(chan struct{})
go func() {
bm.ApplyBackpressure(ctx)
close(done)
}()
// Should complete quickly when disabled
select {
case <-done:
// Success - function returned
case <-time.After(50 * time.Millisecond):
t.Fatal("ApplyBackpressure did not return quickly when disabled")
}
})
t.Run("applies delay when enabled", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true
// Use a channel to verify the function blocks for some time
done := make(chan struct{})
started := make(chan struct{})
go func() {
close(started)
bm.ApplyBackpressure(ctx)
close(done)
}()
// Wait for goroutine to start
<-started
// Should NOT complete immediately - verify it blocks for at least 5ms
select {
case <-done:
t.Fatal("ApplyBackpressure returned too quickly when enabled")
case <-time.After(5 * time.Millisecond):
// Good - it's blocking as expected
}
// Now wait for it to complete (should finish within reasonable time)
select {
case <-done:
// Success - function eventually returned
case <-time.After(500 * time.Millisecond):
t.Fatal("ApplyBackpressure did not complete within timeout")
}
})
t.Run("respects context cancellation", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
start := time.Now()
bm.ApplyBackpressure(ctx)
duration := time.Since(start)
// Should return quickly when context is cancelled
assert.Less(t, duration, 5*time.Millisecond)
})
}
func TestBackpressureManagerLogBackpressureInfo(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true // Ensure enabled so filesProcessed is incremented
// Apply some operations
ctx := context.Background()
bm.ShouldApplyBackpressure(ctx)
bm.ApplyBackpressure(ctx)
// This should not panic
bm.LogBackpressureInfo()
stats := bm.GetStats()
assert.Greater(t, stats.FilesProcessed, int64(0))
}
func TestBackpressureManagerMemoryLimiting(t *testing.T) {
t.Run("triggers on low memory limit", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true
bm.memoryCheckInterval = 1 // Check every file
bm.maxMemoryUsage = 1 // Very low limit to guarantee trigger
ctx := context.Background()
// Should detect memory over limit
shouldApply := bm.ShouldApplyBackpressure(ctx)
assert.True(t, shouldApply)
stats := bm.GetStats()
assert.True(t, stats.MemoryWarningActive)
})
t.Run("resets warning when memory normalizes", func(t *testing.T) {
bm := NewBackpressureManager()
bm.enabled = true
bm.memoryCheckInterval = 1
// Simulate warning by first triggering high memory usage
bm.maxMemoryUsage = 1 // Very low to trigger warning
ctx := context.Background()
_ = bm.ShouldApplyBackpressure(ctx)
stats := bm.GetStats()
assert.True(t, stats.MemoryWarningActive)
// Now set high limit so we're under it
bm.maxMemoryUsage = 1024 * 1024 * 1024 * 10 // 10GB
shouldApply := bm.ShouldApplyBackpressure(ctx)
assert.False(t, shouldApply)
// Warning should be reset (via public API)
stats = bm.GetStats()
assert.False(t, stats.MemoryWarningActive)
})
}

262
fileproc/backpressure_channels_test.go
View File

@@ -1,262 +0,0 @@
package fileproc
import (
"context"
"testing"
"time"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
)
const (
// CI-safe timeout constants
fastOpTimeout = 100 * time.Millisecond // Operations that should complete quickly
slowOpMinTime = 10 * time.Millisecond // Minimum time for blocking operations
)
// cleanupViperConfig is a test helper that captures and restores viper configuration.
// It takes a testing.T and a list of config keys to save/restore.
// Returns a cleanup function that should be called via t.Cleanup.
func cleanupViperConfig(t *testing.T, keys ...string) {
t.Helper()
// Capture original values
origValues := make(map[string]interface{})
for _, key := range keys {
origValues[key] = viper.Get(key)
}
// Register cleanup to restore values
t.Cleanup(func() {
for key, val := range origValues {
if val != nil {
viper.Set(key, val)
}
}
})
}
func TestBackpressureManagerCreateChannels(t *testing.T) {
t.Run("creates buffered channels when enabled", func(t *testing.T) {
// Capture and restore viper config
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles, testBackpressureMaxWrites)
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxFiles, 10)
viper.Set(testBackpressureMaxWrites, 10)
bm := NewBackpressureManager()
fileCh, writeCh := bm.CreateChannels()
assert.NotNil(t, fileCh)
assert.NotNil(t, writeCh)
// Test that channels have buffer capacity
assert.Greater(t, cap(fileCh), 0)
assert.Greater(t, cap(writeCh), 0)
// Test sending and receiving
fileCh <- "test.go"
val := <-fileCh
assert.Equal(t, "test.go", val)
writeCh <- WriteRequest{Content: "test content"}
writeReq := <-writeCh
assert.Equal(t, "test content", writeReq.Content)
close(fileCh)
close(writeCh)
})
t.Run("creates unbuffered channels when disabled", func(t *testing.T) {
// Use viper to configure instead of direct field access
cleanupViperConfig(t, testBackpressureEnabled)
viper.Set(testBackpressureEnabled, false)
bm := NewBackpressureManager()
fileCh, writeCh := bm.CreateChannels()
assert.NotNil(t, fileCh)
assert.NotNil(t, writeCh)
// Unbuffered channels have capacity 0
assert.Equal(t, 0, cap(fileCh))
assert.Equal(t, 0, cap(writeCh))
close(fileCh)
close(writeCh)
})
}
func TestBackpressureManagerWaitForChannelSpace(t *testing.T) {
t.Run("does nothing when disabled", func(t *testing.T) {
// Use viper to configure instead of direct field access
cleanupViperConfig(t, testBackpressureEnabled)
viper.Set(testBackpressureEnabled, false)
bm := NewBackpressureManager()
fileCh := make(chan string, 1)
writeCh := make(chan WriteRequest, 1)
// Use context with timeout instead of measuring elapsed time
ctx, cancel := context.WithTimeout(context.Background(), fastOpTimeout)
defer cancel()
done := make(chan struct{})
go func() {
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
close(done)
}()
// Should return immediately (before timeout)
select {
case <-done:
// Success - operation completed quickly
case <-ctx.Done():
t.Fatal("WaitForChannelSpace should return immediately when disabled")
}
close(fileCh)
close(writeCh)
})
t.Run("waits when file channel is nearly full", func(t *testing.T) {
// Use viper to configure instead of direct field access
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles)
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxFiles, 10)
bm := NewBackpressureManager()
// Create channel with exact capacity
fileCh := make(chan string, 10)
writeCh := make(chan WriteRequest, 10)
// Fill file channel to >90% (with minimum of 1)
target := max(1, int(float64(cap(fileCh))*0.9))
for i := 0; i < target; i++ {
fileCh <- "file.txt"
}
// Test that it blocks by verifying it doesn't complete immediately
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
done := make(chan struct{})
start := time.Now()
go func() {
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
close(done)
}()
// Verify it doesn't complete immediately (within first millisecond)
select {
case <-done:
t.Fatal("WaitForChannelSpace should block when channel is nearly full")
case <-time.After(1 * time.Millisecond):
// Good - it's blocking as expected
}
// Wait for it to complete
<-done
duration := time.Since(start)
// Just verify it took some measurable time (very lenient for CI)
assert.GreaterOrEqual(t, duration, 1*time.Millisecond)
// Clean up
for i := 0; i < target; i++ {
<-fileCh
}
close(fileCh)
close(writeCh)
})
t.Run("waits when write channel is nearly full", func(t *testing.T) {
// Use viper to configure instead of direct field access
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxWrites)
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxWrites, 10)
bm := NewBackpressureManager()
fileCh := make(chan string, 10)
writeCh := make(chan WriteRequest, 10)
// Fill write channel to >90% (with minimum of 1)
target := max(1, int(float64(cap(writeCh))*0.9))
for i := 0; i < target; i++ {
writeCh <- WriteRequest{}
}
// Test that it blocks by verifying it doesn't complete immediately
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
done := make(chan struct{})
start := time.Now()
go func() {
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
close(done)
}()
// Verify it doesn't complete immediately (within first millisecond)
select {
case <-done:
t.Fatal("WaitForChannelSpace should block when channel is nearly full")
case <-time.After(1 * time.Millisecond):
// Good - it's blocking as expected
}
// Wait for it to complete
<-done
duration := time.Since(start)
// Just verify it took some measurable time (very lenient for CI)
assert.GreaterOrEqual(t, duration, 1*time.Millisecond)
// Clean up
for i := 0; i < target; i++ {
<-writeCh
}
close(fileCh)
close(writeCh)
})
t.Run("respects context cancellation", func(t *testing.T) {
// Use viper to configure instead of direct field access
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles)
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxFiles, 10)
bm := NewBackpressureManager()
fileCh := make(chan string, 10)
writeCh := make(chan WriteRequest, 10)
// Fill channel
for i := 0; i < 10; i++ {
fileCh <- "file.txt"
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
// Use timeout to verify it returns quickly
done := make(chan struct{})
go func() {
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
close(done)
}()
// Should return quickly when context is cancelled
select {
case <-done:
// Success - returned due to cancellation
case <-time.After(fastOpTimeout):
t.Fatal("WaitForChannelSpace should return immediately when context is cancelled")
}
// Clean up
for i := 0; i < 10; i++ {
<-fileCh
}
close(fileCh)
close(writeCh)
})
}

195
fileproc/backpressure_concurrency_test.go
View File

@@ -1,195 +0,0 @@
package fileproc
import (
"context"
"sync"
"testing"
"time"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBackpressureManagerConcurrency(t *testing.T) {
// Configure via viper instead of direct field access
origEnabled := viper.Get(testBackpressureEnabled)
t.Cleanup(func() {
if origEnabled != nil {
viper.Set(testBackpressureEnabled, origEnabled)
}
})
viper.Set(testBackpressureEnabled, true)
bm := NewBackpressureManager()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var wg sync.WaitGroup
// Multiple goroutines checking backpressure
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
bm.ShouldApplyBackpressure(ctx)
}()
}
// Multiple goroutines applying backpressure
for i := 0; i < 5; i++ {
wg.Add(1)
go func() {
defer wg.Done()
bm.ApplyBackpressure(ctx)
}()
}
// Multiple goroutines getting stats
for i := 0; i < 5; i++ {
wg.Add(1)
go func() {
defer wg.Done()
bm.GetStats()
}()
}
// Multiple goroutines creating channels
// Note: CreateChannels returns new channels each time, caller owns them
type channelResult struct {
fileCh chan string
writeCh chan WriteRequest
}
results := make(chan channelResult, 3)
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
fileCh, writeCh := bm.CreateChannels()
results <- channelResult{fileCh, writeCh}
}()
}
wg.Wait()
close(results)
// Verify channels are created and have expected properties
for result := range results {
assert.NotNil(t, result.fileCh)
assert.NotNil(t, result.writeCh)
// Close channels to prevent resource leak (caller owns them)
close(result.fileCh)
close(result.writeCh)
}
// Verify stats are consistent
stats := bm.GetStats()
assert.GreaterOrEqual(t, stats.FilesProcessed, int64(10))
}
func TestBackpressureManagerIntegration(t *testing.T) {
// Configure via viper instead of direct field access
origEnabled := viper.Get(testBackpressureEnabled)
origMaxFiles := viper.Get(testBackpressureMaxFiles)
origMaxWrites := viper.Get(testBackpressureMaxWrites)
origCheckInterval := viper.Get(testBackpressureMemoryCheck)
origMaxMemory := viper.Get(testBackpressureMaxMemory)
t.Cleanup(func() {
if origEnabled != nil {
viper.Set(testBackpressureEnabled, origEnabled)
}
if origMaxFiles != nil {
viper.Set(testBackpressureMaxFiles, origMaxFiles)
}
if origMaxWrites != nil {
viper.Set(testBackpressureMaxWrites, origMaxWrites)
}
if origCheckInterval != nil {
viper.Set(testBackpressureMemoryCheck, origCheckInterval)
}
if origMaxMemory != nil {
viper.Set(testBackpressureMaxMemory, origMaxMemory)
}
})
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxFiles, 10)
viper.Set(testBackpressureMaxWrites, 10)
viper.Set(testBackpressureMemoryCheck, 10)
viper.Set(testBackpressureMaxMemory, 100*1024*1024) // 100MB
bm := NewBackpressureManager()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Create channels - caller owns these channels and is responsible for closing them
fileCh, writeCh := bm.CreateChannels()
require.NotNil(t, fileCh)
require.NotNil(t, writeCh)
require.Greater(t, cap(fileCh), 0, "fileCh should be buffered")
require.Greater(t, cap(writeCh), 0, "writeCh should be buffered")
// Simulate file processing
var wg sync.WaitGroup
// Producer
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
// Check for backpressure
if bm.ShouldApplyBackpressure(ctx) {
bm.ApplyBackpressure(ctx)
}
// Wait for channel space if needed
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
select {
case fileCh <- "file.txt":
// File sent
case <-ctx.Done():
return
}
}
}()
// Consumer
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
select {
case <-fileCh:
// Process file (do not manually increment filesProcessed)
case <-ctx.Done():
return
}
}
}()
// Wait for completion
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
// Success
case <-time.After(5 * time.Second):
t.Fatal("Integration test timeout")
}
// Log final info
bm.LogBackpressureInfo()
// Check final stats
stats := bm.GetStats()
assert.GreaterOrEqual(t, stats.FilesProcessed, int64(100))
// Clean up - caller owns the channels, safe to close now that goroutines have finished
close(fileCh)
close(writeCh)
}

151
fileproc/backpressure_init_test.go
View File

@@ -1,151 +0,0 @@
package fileproc
import (
"context"
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
)
// setupViperCleanup is a test helper that captures and restores viper configuration.
// It takes a testing.T and a list of config keys to save/restore.
func setupViperCleanup(t *testing.T, keys []string) {
t.Helper()
// Capture original values and track which keys existed
origValues := make(map[string]interface{})
keysExisted := make(map[string]bool)
for _, key := range keys {
val := viper.Get(key)
origValues[key] = val
keysExisted[key] = viper.IsSet(key)
}
// Register cleanup to restore values
t.Cleanup(func() {
for _, key := range keys {
if keysExisted[key] {
viper.Set(key, origValues[key])
} else {
// Key didn't exist originally, so remove it
allSettings := viper.AllSettings()
delete(allSettings, key)
viper.Reset()
for k, v := range allSettings {
viper.Set(k, v)
}
}
}
})
}
func TestNewBackpressureManager(t *testing.T) {
keys := []string{
testBackpressureEnabled,
testBackpressureMaxMemory,
testBackpressureMemoryCheck,
testBackpressureMaxFiles,
testBackpressureMaxWrites,
}
setupViperCleanup(t, keys)
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxMemory, 100)
viper.Set(testBackpressureMemoryCheck, 10)
viper.Set(testBackpressureMaxFiles, 10)
viper.Set(testBackpressureMaxWrites, 10)
bm := NewBackpressureManager()
assert.NotNil(t, bm)
assert.True(t, bm.enabled)
assert.Greater(t, bm.maxMemoryUsage, int64(0))
assert.Greater(t, bm.memoryCheckInterval, 0)
assert.Greater(t, bm.maxPendingFiles, 0)
assert.Greater(t, bm.maxPendingWrites, 0)
assert.Equal(t, int64(0), bm.filesProcessed)
}
func TestBackpressureStatsStructure(t *testing.T) {
// Behavioral test that exercises BackpressureManager and validates stats
keys := []string{
testBackpressureEnabled,
testBackpressureMaxMemory,
testBackpressureMemoryCheck,
testBackpressureMaxFiles,
testBackpressureMaxWrites,
}
setupViperCleanup(t, keys)
// Configure backpressure with realistic settings
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMaxMemory, 100*1024*1024) // 100MB
viper.Set(testBackpressureMemoryCheck, 1) // Check every file
viper.Set(testBackpressureMaxFiles, 1000)
viper.Set(testBackpressureMaxWrites, 500)
bm := NewBackpressureManager()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Simulate processing files
initialStats := bm.GetStats()
assert.True(t, initialStats.Enabled, "backpressure should be enabled")
assert.Equal(t, int64(0), initialStats.FilesProcessed, "initially no files processed")
// Capture initial timestamp to verify it gets updated
initialLastCheck := initialStats.LastMemoryCheck
// Process some files to trigger memory checks
for i := 0; i < 5; i++ {
bm.ShouldApplyBackpressure(ctx)
}
// Verify stats reflect the operations
stats := bm.GetStats()
assert.True(t, stats.Enabled, "enabled flag should be set")
assert.Equal(t, int64(5), stats.FilesProcessed, "should have processed 5 files")
assert.Greater(t, stats.CurrentMemoryUsage, int64(0), "memory usage should be tracked")
assert.Equal(t, int64(100*1024*1024), stats.MaxMemoryUsage, "max memory should match config")
assert.Equal(t, 1000, stats.MaxPendingFiles, "maxPendingFiles should match config")
assert.Equal(t, 500, stats.MaxPendingWrites, "maxPendingWrites should match config")
assert.True(t, stats.LastMemoryCheck.After(initialLastCheck) || stats.LastMemoryCheck.Equal(initialLastCheck),
"lastMemoryCheck should be updated or remain initialized")
}
func TestBackpressureManagerGetStats(t *testing.T) {
keys := []string{
testBackpressureEnabled,
testBackpressureMemoryCheck,
}
setupViperCleanup(t, keys)
// Ensure config enables backpressure and checks every call
viper.Set(testBackpressureEnabled, true)
viper.Set(testBackpressureMemoryCheck, 1)
bm := NewBackpressureManager()
// Capture initial timestamp to verify it gets updated
initialStats := bm.GetStats()
initialLastCheck := initialStats.LastMemoryCheck
// Process some files to update stats
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for i := 0; i < 5; i++ {
bm.ShouldApplyBackpressure(ctx)
}
stats := bm.GetStats()
assert.True(t, stats.Enabled)
assert.Equal(t, int64(5), stats.FilesProcessed)
assert.Greater(t, stats.CurrentMemoryUsage, int64(0))
assert.Equal(t, bm.maxMemoryUsage, stats.MaxMemoryUsage)
assert.Equal(t, bm.maxPendingFiles, stats.MaxPendingFiles)
assert.Equal(t, bm.maxPendingWrites, stats.MaxPendingWrites)
// LastMemoryCheck should be updated after processing files (memoryCheckInterval=1)
assert.True(t, stats.LastMemoryCheck.After(initialLastCheck),
"lastMemoryCheck should be updated after memory checks")
}

344
fileproc/backpressure_test.go Normal file
View File

@@ -0,0 +1,344 @@
package fileproc_test
import (
"context"
"runtime"
"testing"
"time"
"github.com/ivuorinen/gibidify/fileproc"
"github.com/ivuorinen/gibidify/shared"
"github.com/ivuorinen/gibidify/testutil"
)
func TestNewBackpressureManager(t *testing.T) {
// Test creating a new backpressure manager
bp := fileproc.NewBackpressureManager()
if bp == nil {
t.Error("Expected backpressure manager to be created, got nil")
}
// The backpressure manager should be initialized with config values
// We can't test the internal values directly since they're private,
// but we can test that it was created successfully
}
func TestBackpressureManagerCreateChannels(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
// Test creating channels
fileCh, writeCh := bp.CreateChannels()
// Verify channels are created
if fileCh == nil {
t.Error("Expected file channel to be created, got nil")
}
if writeCh == nil {
t.Error("Expected write channel to be created, got nil")
}
// Test that channels can be used
select {
case fileCh <- "test-file":
// Successfully sent to channel
default:
t.Error("Unable to send to file channel")
}
// Read from channel
select {
case file := <-fileCh:
if file != "test-file" {
t.Errorf("Expected 'test-file', got %s", file)
}
case <-time.After(100 * time.Millisecond):
t.Error("Timeout reading from file channel")
}
}
func TestBackpressureManagerShouldApplyBackpressure(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
// Test backpressure decision
shouldApply := bp.ShouldApplyBackpressure(ctx)
// Since we're using default config, backpressure behavior depends on settings
// We just test that the method returns without error
// shouldApply is a valid boolean value
_ = shouldApply
}
func TestBackpressureManagerApplyBackpressure(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
// Test applying backpressure
bp.ApplyBackpressure(ctx)
// ApplyBackpressure is a void method that should not panic
// If we reach here, the method executed successfully
}
func TestBackpressureManagerApplyBackpressureWithCancellation(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
// Create canceled context
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
// Test applying backpressure with canceled context
bp.ApplyBackpressure(ctx)
// ApplyBackpressure doesn't return errors, but should handle cancellation gracefully
// If we reach here without hanging, the method handled cancellation properly
}
func TestBackpressureManagerGetStats(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
// Test getting stats
stats := bp.Stats()
// Stats should contain relevant information
if stats.FilesProcessed < 0 {
t.Error("Expected non-negative files processed count")
}
if stats.CurrentMemoryUsage < 0 {
t.Error("Expected non-negative memory usage")
}
if stats.MaxMemoryUsage < 0 {
t.Error("Expected non-negative max memory usage")
}
// Test that stats have reasonable values
if stats.MaxPendingFiles < 0 || stats.MaxPendingWrites < 0 {
t.Error("Expected non-negative channel buffer sizes")
}
}
func TestBackpressureManagerWaitForChannelSpace(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
// Create test channels
fileCh, writeCh := bp.CreateChannels()
// Test waiting for channel space
bp.WaitForChannelSpace(ctx, fileCh, writeCh)
// WaitForChannelSpace is void method that should complete without hanging
// If we reach here, the method executed successfully
}
func TestBackpressureManagerWaitForChannelSpaceWithCancellation(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
// Create canceled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
// Create test channels
fileCh, writeCh := bp.CreateChannels()
// Test waiting for channel space with canceled context
bp.WaitForChannelSpace(ctx, fileCh, writeCh)
// WaitForChannelSpace should handle cancellation gracefully without hanging
// If we reach here, the method handled cancellation properly
}
func TestBackpressureManagerLogBackpressureInfo(t *testing.T) {
testutil.ResetViperConfig(t, "")
bp := fileproc.NewBackpressureManager()
// Test logging backpressure info
// This method primarily logs information, so we test it executes without panic
bp.LogBackpressureInfo()
// If we reach here without panic, the method worked
}
// BenchmarkBackpressureManager benchmarks backpressure operations.
func BenchmarkBackpressureManagerCreateChannels(b *testing.B) {
bp := fileproc.NewBackpressureManager()
b.ResetTimer()
for i := 0; i < b.N; i++ {
fileCh, writeCh := bp.CreateChannels()
// Use channels to prevent optimization
_ = fileCh
_ = writeCh
runtime.GC() // Force GC to measure memory impact
}
}
func BenchmarkBackpressureManagerShouldApplyBackpressure(b *testing.B) {
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
b.ResetTimer()
for i := 0; i < b.N; i++ {
shouldApply := bp.ShouldApplyBackpressure(ctx)
_ = shouldApply // Prevent optimization
}
}
func BenchmarkBackpressureManagerApplyBackpressure(b *testing.B) {
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bp.ApplyBackpressure(ctx)
}
}
func BenchmarkBackpressureManagerGetStats(b *testing.B) {
bp := fileproc.NewBackpressureManager()
b.ResetTimer()
for i := 0; i < b.N; i++ {
stats := bp.Stats()
_ = stats // Prevent optimization
}
}
// TestBackpressureManager_ShouldApplyBackpressure_EdgeCases tests various edge cases for backpressure decision.
func TestBackpressureManagerShouldApplyBackpressureEdgeCases(t *testing.T) {
testutil.ApplyBackpressureOverrides(t, map[string]any{
shared.ConfigKeyBackpressureEnabled: true,
"backpressure.memory_check_interval": 2,
"backpressure.memory_limit_mb": 1,
})
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
// Test multiple calls to trigger memory check interval logic
for i := 0; i < 10; i++ {
shouldApply := bp.ShouldApplyBackpressure(ctx)
_ = shouldApply
}
// At this point, memory checking should have triggered multiple times
// The actual decision depends on memory usage, but we're testing the paths
}
// TestBackpressureManager_CreateChannels_EdgeCases tests edge cases in channel creation.
func TestBackpressureManagerCreateChannelsEdgeCases(t *testing.T) {
// Test with custom configuration that might trigger different buffer sizes
testutil.ApplyBackpressureOverrides(t, map[string]any{
"backpressure.file_buffer_size": 50,
"backpressure.write_buffer_size": 25,
})
bp := fileproc.NewBackpressureManager()
// Create multiple channel sets to test resource management
for i := 0; i < 5; i++ {
fileCh, writeCh := bp.CreateChannels()
// Verify channels work correctly
select {
case fileCh <- "test":
// Good - channel accepted value
default:
// This is also acceptable if buffer is full
}
// Test write channel
select {
case writeCh <- fileproc.WriteRequest{Path: "test", Content: "content"}:
// Good - channel accepted value
default:
// This is also acceptable if buffer is full
}
}
}
// TestBackpressureManager_WaitForChannelSpace_EdgeCases tests edge cases in channel space waiting.
func TestBackpressureManagerWaitForChannelSpaceEdgeCases(t *testing.T) {
testutil.ApplyBackpressureOverrides(t, map[string]any{
shared.ConfigKeyBackpressureEnabled: true,
"backpressure.wait_timeout_ms": 10,
})
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
// Create channels with small buffers
fileCh, writeCh := bp.CreateChannels()
// Fill up the channels to create pressure
go func() {
for i := 0; i < 100; i++ {
select {
case fileCh <- "file":
case <-time.After(1 * time.Millisecond):
}
}
}()
go func() {
for i := 0; i < 100; i++ {
select {
case writeCh <- fileproc.WriteRequest{Path: "test", Content: "content"}:
case <-time.After(1 * time.Millisecond):
}
}
}()
// Wait for channel space - should handle the full channels
bp.WaitForChannelSpace(ctx, fileCh, writeCh)
}
// TestBackpressureManager_MemoryPressure tests behavior under simulated memory pressure.
func TestBackpressureManagerMemoryPressure(t *testing.T) {
// Test with very low memory limit to trigger backpressure
testutil.ApplyBackpressureOverrides(t, map[string]any{
shared.ConfigKeyBackpressureEnabled: true,
"backpressure.memory_limit_mb": 0.001,
"backpressure.memory_check_interval": 1,
})
bp := fileproc.NewBackpressureManager()
ctx := context.Background()
// Allocate some memory to potentially trigger limits
largeBuffer := make([]byte, 1024*1024) // 1MB
_ = largeBuffer[0]
// Test backpressure decision under memory pressure
for i := 0; i < 5; i++ {
shouldApply := bp.ShouldApplyBackpressure(ctx)
if shouldApply {
// Test applying backpressure when needed
bp.ApplyBackpressure(ctx)
t.Log("Backpressure applied due to memory pressure")
}
}
// Test logging
bp.LogBackpressureInfo()
}

3
fileproc/cache.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
// getNormalizedExtension efficiently extracts and normalizes the file extension with caching.
@@ -6,6 +7,7 @@ func (r *FileTypeRegistry) getNormalizedExtension(filename string) string {
r.cacheMutex.RLock()
if ext, exists := r.extCache[filename]; exists {
r.cacheMutex.RUnlock()
return ext
}
r.cacheMutex.RUnlock()
@@ -42,6 +44,7 @@ func (r *FileTypeRegistry) getFileTypeResult(filename string) FileTypeResult {
r.updateStats(func() {
r.stats.CacheHits++
})
return result
}
r.cacheMutex.RUnlock()

1
fileproc/collector.go
View File

@@ -5,5 +5,6 @@ package fileproc
// and returns a slice of file paths.
func CollectFiles(root string) ([]string, error) {
w := NewProdWalker()
return w.Walk(root)
}

68
fileproc/collector_test.go
View File

@@ -2,6 +2,7 @@ package fileproc_test
import (
"os"
"path/filepath"
"testing"
"github.com/ivuorinen/gibidify/fileproc"
@@ -47,3 +48,70 @@ func TestCollectFilesError(t *testing.T) {
t.Fatal("Expected an error, got nil")
}
}
// TestCollectFiles tests the actual CollectFiles function with a real directory.
func TestCollectFiles(t *testing.T) {
// Create a temporary directory with test files
tmpDir := t.TempDir()
// Create test files with known supported extensions
testFiles := map[string]string{
"test1.go": "package main\n\nfunc main() {\n\t// Go file\n}",
"test2.py": "# Python file\nprint('hello world')",
"test3.js": "// JavaScript file\nconsole.log('hello');",
}
for name, content := range testFiles {
filePath := filepath.Join(tmpDir, name)
if err := os.WriteFile(filePath, []byte(content), 0o600); err != nil {
t.Fatalf("Failed to create test file %s: %v", name, err)
}
}
// Test CollectFiles
files, err := fileproc.CollectFiles(tmpDir)
if err != nil {
t.Fatalf("CollectFiles failed: %v", err)
}
// Verify we got the expected number of files
if len(files) != len(testFiles) {
t.Errorf("Expected %d files, got %d", len(testFiles), len(files))
}
// Verify all expected files are found
foundFiles := make(map[string]bool)
for _, file := range files {
foundFiles[file] = true
}
for expectedFile := range testFiles {
expectedPath := filepath.Join(tmpDir, expectedFile)
if !foundFiles[expectedPath] {
t.Errorf("Expected file %s not found in results", expectedPath)
}
}
}
// TestCollectFiles_NonExistentDirectory tests CollectFiles with a non-existent directory.
func TestCollectFilesNonExistentDirectory(t *testing.T) {
_, err := fileproc.CollectFiles("/non/existent/directory")
if err == nil {
t.Error("Expected error for non-existent directory, got nil")
}
}
// TestCollectFiles_EmptyDirectory tests CollectFiles with an empty directory.
func TestCollectFilesEmptyDirectory(t *testing.T) {
tmpDir := t.TempDir()
// Don't create any files
files, err := fileproc.CollectFiles(tmpDir)
if err != nil {
t.Fatalf("CollectFiles failed on empty directory: %v", err)
}
if len(files) != 0 {
t.Errorf("Expected 0 files in empty directory, got %d", len(files))
}
}

181
fileproc/config.go
View File

@@ -1,156 +1,7 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
"fmt"
"path/filepath"
"strings"
)
const (
// MaxRegistryEntries is the maximum number of entries allowed in registry config slices/maps.
MaxRegistryEntries = 1000
// MaxExtensionLength is the maximum length for a single extension string.
MaxExtensionLength = 100
)
// RegistryConfig holds configuration for file type registry.
// All paths must be relative without path traversal (no ".." or leading "/").
// Extensions in CustomLanguages keys must start with "." or be alphanumeric with underscore/hyphen.
type RegistryConfig struct {
// CustomImages: file extensions to treat as images (e.g., ".svg", ".webp").
// Must be relative paths without ".." or leading separators.
CustomImages []string
// CustomBinary: file extensions to treat as binary (e.g., ".bin", ".dat").
// Must be relative paths without ".." or leading separators.
CustomBinary []string
// CustomLanguages: maps file extensions to language names (e.g., {".tsx": "TypeScript"}).
// Keys must start with "." or be alphanumeric with underscore/hyphen.
CustomLanguages map[string]string
// DisabledImages: image extensions to disable from default registry.
DisabledImages []string
// DisabledBinary: binary extensions to disable from default registry.
DisabledBinary []string
// DisabledLanguages: language extensions to disable from default registry.
DisabledLanguages []string
}
// Validate checks the RegistryConfig for invalid entries and enforces limits.
func (c *RegistryConfig) Validate() error {
// Validate CustomImages
if err := validateExtensionSlice(c.CustomImages, "CustomImages"); err != nil {
return err
}
// Validate CustomBinary
if err := validateExtensionSlice(c.CustomBinary, "CustomBinary"); err != nil {
return err
}
// Validate CustomLanguages
if len(c.CustomLanguages) > MaxRegistryEntries {
return fmt.Errorf(
"CustomLanguages exceeds maximum entries (%d > %d)",
len(c.CustomLanguages),
MaxRegistryEntries,
)
}
for ext, lang := range c.CustomLanguages {
if err := validateExtension(ext, "CustomLanguages key"); err != nil {
return err
}
if len(lang) > MaxExtensionLength {
return fmt.Errorf(
"CustomLanguages value %q exceeds maximum length (%d > %d)",
lang,
len(lang),
MaxExtensionLength,
)
}
}
// Validate Disabled slices
if err := validateExtensionSlice(c.DisabledImages, "DisabledImages"); err != nil {
return err
}
if err := validateExtensionSlice(c.DisabledBinary, "DisabledBinary"); err != nil {
return err
}
return validateExtensionSlice(c.DisabledLanguages, "DisabledLanguages")
}
// validateExtensionSlice validates a slice of extensions for path safety and limits.
func validateExtensionSlice(slice []string, fieldName string) error {
if len(slice) > MaxRegistryEntries {
return fmt.Errorf("%s exceeds maximum entries (%d > %d)", fieldName, len(slice), MaxRegistryEntries)
}
for _, ext := range slice {
if err := validateExtension(ext, fieldName); err != nil {
return err
}
}
return nil
}
// validateExtension validates a single extension for path safety.
//
//revive:disable-next-line:cyclomatic
func validateExtension(ext, context string) error {
// Reject empty strings
if ext == "" {
return fmt.Errorf("%s entry cannot be empty", context)
}
if len(ext) > MaxExtensionLength {
return fmt.Errorf(
"%s entry %q exceeds maximum length (%d > %d)",
context, ext, len(ext), MaxExtensionLength,
)
}
// Reject absolute paths
if filepath.IsAbs(ext) {
return fmt.Errorf("%s entry %q is an absolute path (not allowed)", context, ext)
}
// Reject path traversal
if strings.Contains(ext, "..") {
return fmt.Errorf("%s entry %q contains path traversal (not allowed)", context, ext)
}
// For extensions, verify they start with "." or are alphanumeric
if strings.HasPrefix(ext, ".") {
// Reject extensions containing path separators
if strings.ContainsRune(ext, filepath.Separator) || strings.ContainsRune(ext, '/') ||
strings.ContainsRune(ext, '\\') {
return fmt.Errorf("%s entry %q contains path separators (not allowed)", context, ext)
}
// Valid extension format
return nil
}
// Check if purely alphanumeric (for bare names)
for _, r := range ext {
isValid := (r >= 'a' && r <= 'z') ||
(r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') ||
r == '_' || r == '-'
if !isValid {
return fmt.Errorf(
"%s entry %q contains invalid characters (must start with '.' or be alphanumeric/_/-)",
context,
ext,
)
}
}
return nil
}
import "strings"
// ApplyCustomExtensions applies custom extensions from configuration.
func (r *FileTypeRegistry) ApplyCustomExtensions(
@@ -182,24 +33,12 @@ func (r *FileTypeRegistry) addExtensions(extensions []string, adder func(string)
// ConfigureFromSettings applies configuration settings to the registry.
// This function is called from main.go after config is loaded to avoid circular imports.
// It validates the configuration before applying it.
func ConfigureFromSettings(config RegistryConfig) error {
// Validate configuration first
if err := config.Validate(); err != nil {
return err
}
registry := GetDefaultRegistry()
// Only apply custom extensions if they are non-empty (len() for nil slices/maps is zero)
if len(config.CustomImages) > 0 || len(config.CustomBinary) > 0 || len(config.CustomLanguages) > 0 {
registry.ApplyCustomExtensions(config.CustomImages, config.CustomBinary, config.CustomLanguages)
}
// Only disable extensions if they are non-empty
if len(config.DisabledImages) > 0 || len(config.DisabledBinary) > 0 || len(config.DisabledLanguages) > 0 {
registry.DisableExtensions(config.DisabledImages, config.DisabledBinary, config.DisabledLanguages)
}
return nil
func ConfigureFromSettings(
customImages, customBinary []string,
customLanguages map[string]string,
disabledImages, disabledBinary, disabledLanguages []string,
) {
registry := DefaultRegistry()
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
registry.DisableExtensions(disabledImages, disabledBinary, disabledLanguages)
}

14
fileproc/detection.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import "strings"
@@ -14,9 +15,9 @@ func IsBinary(filename string) bool {
return getRegistry().IsBinary(filename)
}
// GetLanguage returns the language identifier for the given filename based on its extension.
func GetLanguage(filename string) string {
return getRegistry().GetLanguage(filename)
// Language returns the language identifier for the given filename based on its extension.
func Language(filename string) string {
return getRegistry().Language(filename)
}
// Registry methods for detection
@@ -24,21 +25,24 @@ func GetLanguage(filename string) string {
// IsImage checks if the file extension indicates an image file.
func (r *FileTypeRegistry) IsImage(filename string) bool {
result := r.getFileTypeResult(filename)
return result.IsImage
}
// IsBinary checks if the file extension indicates a binary file.
func (r *FileTypeRegistry) IsBinary(filename string) bool {
result := r.getFileTypeResult(filename)
return result.IsBinary
}
// GetLanguage returns the language identifier for the given filename based on its extension.
func (r *FileTypeRegistry) GetLanguage(filename string) string {
// Language returns the language identifier for the given filename based on its extension.
func (r *FileTypeRegistry) Language(filename string) string {
if len(filename) < minExtensionLength {
return ""
}
result := r.getFileTypeResult(filename)
return result.Language
}

11
fileproc/extensions.go
View File

@@ -1,5 +1,8 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import "github.com/ivuorinen/gibidify/shared"
// getImageExtensions returns the default image file extensions.
func getImageExtensions() map[string]bool {
return map[string]bool{
@@ -130,15 +133,15 @@ func getLanguageMap() map[string]string {
".cmd": "batch",
// Data formats
".json": "json",
".yaml": "yaml",
".yml": "yaml",
".json": shared.FormatJSON,
".yaml": shared.FormatYAML,
".yml": shared.FormatYAML,
".toml": "toml",
".xml": "xml",
".sql": "sql",
// Documentation
".md": "markdown",
".md": shared.FormatMarkdown,
".rst": "rst",
".tex": "latex",

1
fileproc/fake_walker.go
View File

@@ -12,5 +12,6 @@ func (fw FakeWalker) Walk(_ string) ([]string, error) {
if fw.Err != nil {
return nil, fw.Err
}
return fw.Files, nil
}

6
fileproc/file_filters.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
@@ -15,8 +16,8 @@ type FileFilter struct {
// NewFileFilter creates a new file filter with current configuration.
func NewFileFilter() *FileFilter {
return &FileFilter{
ignoredDirs: config.GetIgnoredDirectories(),
sizeLimit: config.GetFileSizeLimit(),
ignoredDirs: config.IgnoredDirectories(),
sizeLimit: config.FileSizeLimit(),
}
}
@@ -40,6 +41,7 @@ func (f *FileFilter) shouldSkipDirectory(entry os.DirEntry) bool {
return true
}
}
return false
}

265
fileproc/filetypes_concurrency_test.go
View File

@@ -1,105 +1,200 @@
package fileproc
import (
"errors"
"fmt"
"sync"
"testing"
"github.com/ivuorinen/gibidify/shared"
)
// TestFileTypeRegistry_ThreadSafety tests thread safety of the FileTypeRegistry.
func TestFileTypeRegistry_ThreadSafety(t *testing.T) {
const numGoroutines = 100
const numOperationsPerGoroutine = 100
const (
numGoroutines = 100
numOperationsPerGoroutine = 100
)
// TestFileTypeRegistryConcurrentReads tests concurrent read operations.
// This test verifies thread-safety of registry reads under concurrent access.
// For race condition detection, run with: go test -race
func TestFileTypeRegistryConcurrentReads(t *testing.T) {
var wg sync.WaitGroup
errChan := make(chan error, numGoroutines)
for i := 0; i < numGoroutines; i++ {
wg.Go(func() {
if err := performConcurrentReads(); err != nil {
errChan <- err
}
})
}
wg.Wait()
close(errChan)
// Check for any errors from goroutines
for err := range errChan {
t.Errorf("Concurrent read operation failed: %v", err)
}
}
// TestFileTypeRegistryConcurrentRegistryAccess tests concurrent registry access.
func TestFileTypeRegistryConcurrentRegistryAccess(t *testing.T) {
// Reset the registry to test concurrent initialization
ResetRegistryForTesting()
t.Cleanup(func() {
ResetRegistryForTesting()
})
registries := make([]*FileTypeRegistry, numGoroutines)
var wg sync.WaitGroup
// Test concurrent read operations
t.Run("ConcurrentReads", func(_ *testing.T) {
for i := 0; i < numGoroutines; i++ {
wg.Add(1)
go func(_ int) {
defer wg.Done()
registry := GetDefaultRegistry()
for i := 0; i < numGoroutines; i++ {
idx := i // capture for closure
wg.Go(func() {
registries[idx] = DefaultRegistry()
})
}
wg.Wait()
for j := 0; j < numOperationsPerGoroutine; j++ {
// Test various file detection operations
_ = registry.IsImage("test.png")
_ = registry.IsBinary("test.exe")
_ = registry.GetLanguage("test.go")
verifySameRegistryInstance(t, registries)
}
// Test global functions too
_ = IsImage("image.jpg")
_ = IsBinary("binary.dll")
_ = GetLanguage("script.py")
}
}(i)
// TestFileTypeRegistryConcurrentModifications tests concurrent modifications.
func TestFileTypeRegistryConcurrentModifications(t *testing.T) {
var wg sync.WaitGroup
for i := 0; i < numGoroutines; i++ {
id := i // capture for closure
wg.Go(func() {
performConcurrentModifications(t, id)
})
}
wg.Wait()
}
// performConcurrentReads performs concurrent read operations on the registry.
// Returns an error if any operation produces unexpected results.
func performConcurrentReads() error {
registry := DefaultRegistry()
for j := 0; j < numOperationsPerGoroutine; j++ {
// Test various file detection operations with expected results
if !registry.IsImage(shared.TestFilePNG) {
return errors.New("expected .png to be detected as image")
}
wg.Wait()
})
// Test concurrent registry access (singleton creation)
t.Run("ConcurrentRegistryAccess", func(t *testing.T) {
// Reset the registry to test concurrent initialization
// Note: This is not safe in a real application, but needed for testing
registryOnce = sync.Once{}
registry = nil
registries := make([]*FileTypeRegistry, numGoroutines)
for i := 0; i < numGoroutines; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
registries[id] = GetDefaultRegistry()
}(i)
if !registry.IsBinary(shared.TestFileEXE) {
return errors.New("expected .exe to be detected as binary")
}
wg.Wait()
// Verify all goroutines got the same registry instance
firstRegistry := registries[0]
for i := 1; i < numGoroutines; i++ {
if registries[i] != firstRegistry {
t.Errorf("Registry %d is different from registry 0", i)
}
if lang := registry.Language(shared.TestFileGo); lang != "go" {
return fmt.Errorf("expected .go to have language 'go', got %q", lang)
}
})
// Test concurrent modifications on separate registry instances
t.Run("ConcurrentModifications", func(t *testing.T) {
// Create separate registry instances for each goroutine to test modification thread safety
for i := 0; i < numGoroutines; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
// Create a new registry instance for this goroutine
registry := &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
}
for j := 0; j < numOperationsPerGoroutine; j++ {
// Add unique extensions for this goroutine
extSuffix := fmt.Sprintf("_%d_%d", id, j)
registry.AddImageExtension(".img" + extSuffix)
registry.AddBinaryExtension(".bin" + extSuffix)
registry.AddLanguageMapping(".lang"+extSuffix, "lang"+extSuffix)
// Verify the additions worked
if !registry.IsImage("test.img" + extSuffix) {
t.Errorf("Failed to add image extension .img%s", extSuffix)
}
if !registry.IsBinary("test.bin" + extSuffix) {
t.Errorf("Failed to add binary extension .bin%s", extSuffix)
}
if registry.GetLanguage("test.lang"+extSuffix) != "lang"+extSuffix {
t.Errorf("Failed to add language mapping .lang%s", extSuffix)
}
}
}(i)
// Test global functions with expected results
if !IsImage(shared.TestFileImageJPG) {
return errors.New("expected .jpg to be detected as image")
}
if !IsBinary(shared.TestFileBinaryDLL) {
return errors.New("expected .dll to be detected as binary")
}
if lang := Language(shared.TestFileScriptPy); lang != "python" {
return fmt.Errorf("expected .py to have language 'python', got %q", lang)
}
}
return nil
}
// verifySameRegistryInstance verifies all goroutines got the same registry instance.
func verifySameRegistryInstance(t *testing.T, registries []*FileTypeRegistry) {
t.Helper()
firstRegistry := registries[0]
for i := 1; i < numGoroutines; i++ {
if registries[i] != firstRegistry {
t.Errorf("Registry %d is different from registry 0", i)
}
}
}
// performConcurrentModifications performs concurrent modifications on separate registry instances.
func performConcurrentModifications(t *testing.T, id int) {
t.Helper()
// Create a new registry instance for this goroutine
registry := createConcurrencyTestRegistry()
for j := 0; j < numOperationsPerGoroutine; j++ {
extSuffix := fmt.Sprintf("_%d_%d", id, j)
addTestExtensions(registry, extSuffix)
verifyTestExtensions(t, registry, extSuffix)
}
}
// createConcurrencyTestRegistry creates a new registry instance for concurrency testing.
func createConcurrencyTestRegistry() *FileTypeRegistry {
return &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
extCache: make(map[string]string, shared.FileTypeRegistryMaxCacheSize),
resultCache: make(map[string]FileTypeResult, shared.FileTypeRegistryMaxCacheSize),
maxCacheSize: shared.FileTypeRegistryMaxCacheSize,
}
}
// addTestExtensions adds test extensions to the registry.
func addTestExtensions(registry *FileTypeRegistry, extSuffix string) {
registry.AddImageExtension(".img" + extSuffix)
registry.AddBinaryExtension(".bin" + extSuffix)
registry.AddLanguageMapping(".lang"+extSuffix, "lang"+extSuffix)
}
// verifyTestExtensions verifies that test extensions were added correctly.
func verifyTestExtensions(t *testing.T, registry *FileTypeRegistry, extSuffix string) {
t.Helper()
if !registry.IsImage("test.img" + extSuffix) {
t.Errorf("Failed to add image extension .img%s", extSuffix)
}
if !registry.IsBinary("test.bin" + extSuffix) {
t.Errorf("Failed to add binary extension .bin%s", extSuffix)
}
if registry.Language("test.lang"+extSuffix) != "lang"+extSuffix {
t.Errorf("Failed to add language mapping .lang%s", extSuffix)
}
}
// Benchmarks for concurrency performance
// BenchmarkConcurrentReads benchmarks concurrent read operations on the registry.
func BenchmarkConcurrentReads(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = performConcurrentReads()
}
wg.Wait()
})
}
// BenchmarkConcurrentRegistryAccess benchmarks concurrent registry singleton access.
func BenchmarkConcurrentRegistryAccess(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = DefaultRegistry()
}
})
}
// BenchmarkConcurrentModifications benchmarks sequential registry modifications.
// Note: Concurrent modifications to the same registry require external synchronization.
// This benchmark measures the cost of modification operations themselves.
func BenchmarkConcurrentModifications(b *testing.B) {
for b.Loop() {
registry := createConcurrencyTestRegistry()
for i := 0; i < 10; i++ {
extSuffix := fmt.Sprintf("_bench_%d", i)
registry.AddImageExtension(".img" + extSuffix)
registry.AddBinaryExtension(".bin" + extSuffix)
registry.AddLanguageMapping(".lang"+extSuffix, "lang"+extSuffix)
}
}
}

410
fileproc/filetypes_config_test.go
View File

@@ -3,218 +3,264 @@ package fileproc
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/ivuorinen/gibidify/shared"
)
// TestFileTypeRegistry_Configuration tests the configuration functionality.
func TestFileTypeRegistry_Configuration(t *testing.T) {
// Create a new registry instance for testing
registry := &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
const (
zigLang = "zig"
)
// TestFileTypeRegistryApplyCustomExtensions tests applying custom extensions.
func TestFileTypeRegistryApplyCustomExtensions(t *testing.T) {
registry := createEmptyTestRegistry()
customImages := []string{".webp", ".avif", ".heic"}
customBinary := []string{".custom", ".mybin"}
customLanguages := map[string]string{
".zig": zigLang,
".odin": "odin",
".v": "vlang",
}
// Test ApplyCustomExtensions
t.Run("ApplyCustomExtensions", func(t *testing.T) {
customImages := []string{".webp", ".avif", ".heic"}
customBinary := []string{".custom", ".mybin"}
customLanguages := map[string]string{
".zig": "zig",
".odin": "odin",
".v": "vlang",
}
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
verifyCustomExtensions(t, registry, customImages, customBinary, customLanguages)
}
// Test custom image extensions
for _, ext := range customImages {
if !registry.IsImage("test" + ext) {
t.Errorf("Expected %s to be recognized as image", ext)
}
}
// TestFileTypeRegistryDisableExtensions tests disabling extensions.
func TestFileTypeRegistryDisableExtensions(t *testing.T) {
registry := createEmptyTestRegistry()
// Test custom binary extensions
for _, ext := range customBinary {
if !registry.IsBinary("test" + ext) {
t.Errorf("Expected %s to be recognized as binary", ext)
}
}
// Add some extensions first
setupRegistryExtensions(registry)
// Test custom language mappings
for ext, expectedLang := range customLanguages {
if lang := registry.GetLanguage("test" + ext); lang != expectedLang {
t.Errorf("Expected %s to map to %s, got %s", ext, expectedLang, lang)
}
}
})
// Verify they work before disabling
verifyExtensionsEnabled(t, registry)
// Test DisableExtensions
t.Run("DisableExtensions", func(t *testing.T) {
// Add some extensions first
registry.AddImageExtension(".png")
registry.AddImageExtension(".jpg")
registry.AddBinaryExtension(".exe")
registry.AddBinaryExtension(".dll")
registry.AddLanguageMapping(".go", "go")
registry.AddLanguageMapping(".py", "python")
// Disable some extensions
disabledImages := []string{".png"}
disabledBinary := []string{".exe"}
disabledLanguages := []string{".go"}
// Verify they work
if !registry.IsImage("test.png") {
t.Error("Expected .png to be image before disabling")
}
if !registry.IsBinary("test.exe") {
t.Error("Expected .exe to be binary before disabling")
}
if registry.GetLanguage("test.go") != "go" {
t.Error("Expected .go to map to go before disabling")
}
registry.DisableExtensions(disabledImages, disabledBinary, disabledLanguages)
// Disable some extensions
disabledImages := []string{".png"}
disabledBinary := []string{".exe"}
disabledLanguages := []string{".go"}
// Verify disabled and remaining extensions
verifyExtensionsDisabled(t, registry)
verifyRemainingExtensions(t, registry)
}
registry.DisableExtensions(disabledImages, disabledBinary, disabledLanguages)
// TestFileTypeRegistryEmptyValuesHandling tests handling of empty values.
func TestFileTypeRegistryEmptyValuesHandling(t *testing.T) {
registry := createEmptyTestRegistry()
// Test that disabled extensions no longer work
if registry.IsImage("test.png") {
t.Error("Expected .png to not be image after disabling")
}
if registry.IsBinary("test.exe") {
t.Error("Expected .exe to not be binary after disabling")
}
if registry.GetLanguage("test.go") != "" {
t.Error("Expected .go to not map to language after disabling")
}
customImages := []string{"", shared.TestExtensionValid, ""}
customBinary := []string{"", shared.TestExtensionValid}
customLanguages := map[string]string{
"": "invalid",
shared.TestExtensionValid: "",
".good": "good",
}
// Test that non-disabled extensions still work
if !registry.IsImage("test.jpg") {
t.Error("Expected .jpg to still be image after disabling .png")
}
if !registry.IsBinary("test.dll") {
t.Error("Expected .dll to still be binary after disabling .exe")
}
if registry.GetLanguage("test.py") != "python" {
t.Error("Expected .py to still map to python after disabling .go")
}
})
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
// Test empty values handling
t.Run("EmptyValuesHandling", func(t *testing.T) {
registry := &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
}
verifyEmptyValueHandling(t, registry)
}
// Test with empty values
customImages := []string{"", ".valid", ""}
customBinary := []string{"", ".valid"}
customLanguages := map[string]string{
"": "invalid",
".valid": "",
".good": "good",
}
// TestFileTypeRegistryCaseInsensitiveHandling tests case insensitive handling.
func TestFileTypeRegistryCaseInsensitiveHandling(t *testing.T) {
registry := createEmptyTestRegistry()
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
customImages := []string{".WEBP", ".Avif"}
customBinary := []string{".CUSTOM", ".MyBin"}
customLanguages := map[string]string{
".ZIG": zigLang,
".Odin": "odin",
}
// Only valid entries should be added
if registry.IsImage("test.") {
t.Error("Expected empty extension to not be added as image")
}
if !registry.IsImage("test.valid") {
t.Error("Expected .valid to be added as image")
}
if registry.IsBinary("test.") {
t.Error("Expected empty extension to not be added as binary")
}
if !registry.IsBinary("test.valid") {
t.Error("Expected .valid to be added as binary")
}
if registry.GetLanguage("test.") != "" {
t.Error("Expected empty extension to not be added as language")
}
if registry.GetLanguage("test.valid") != "" {
t.Error("Expected .valid with empty language to not be added")
}
if registry.GetLanguage("test.good") != "good" {
t.Error("Expected .good to map to good")
}
})
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
// Test case-insensitive handling
t.Run("CaseInsensitiveHandling", func(t *testing.T) {
registry := &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
}
verifyCaseInsensitiveHandling(t, registry)
}
customImages := []string{".WEBP", ".Avif"}
customBinary := []string{".CUSTOM", ".MyBin"}
customLanguages := map[string]string{
".ZIG": "zig",
".Odin": "odin",
}
// createEmptyTestRegistry creates a new empty test registry instance for config testing.
func createEmptyTestRegistry() *FileTypeRegistry {
return &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
extCache: make(map[string]string, shared.FileTypeRegistryMaxCacheSize),
resultCache: make(map[string]FileTypeResult, shared.FileTypeRegistryMaxCacheSize),
maxCacheSize: shared.FileTypeRegistryMaxCacheSize,
}
}
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
// verifyCustomExtensions verifies that custom extensions are applied correctly.
func verifyCustomExtensions(
t *testing.T,
registry *FileTypeRegistry,
customImages, customBinary []string,
customLanguages map[string]string,
) {
t.Helper()
// Test that both upper and lower case work
if !registry.IsImage("test.webp") {
t.Error("Expected .webp (lowercase) to work after adding .WEBP")
// Test custom image extensions
for _, ext := range customImages {
if !registry.IsImage("test" + ext) {
t.Errorf("Expected %s to be recognized as image", ext)
}
if !registry.IsImage("test.WEBP") {
t.Error("Expected .WEBP (uppercase) to work")
}
// Test custom binary extensions
for _, ext := range customBinary {
if !registry.IsBinary("test" + ext) {
t.Errorf("Expected %s to be recognized as binary", ext)
}
if !registry.IsBinary("test.custom") {
t.Error("Expected .custom (lowercase) to work after adding .CUSTOM")
}
// Test custom language mappings
for ext, expectedLang := range customLanguages {
if lang := registry.Language("test" + ext); lang != expectedLang {
t.Errorf("Expected %s to map to %s, got %s", ext, expectedLang, lang)
}
if !registry.IsBinary("test.CUSTOM") {
t.Error("Expected .CUSTOM (uppercase) to work")
}
if registry.GetLanguage("test.zig") != "zig" {
t.Error("Expected .zig (lowercase) to work after adding .ZIG")
}
if registry.GetLanguage("test.ZIG") != "zig" {
t.Error("Expected .ZIG (uppercase) to work")
}
})
}
}
// setupRegistryExtensions adds test extensions to the registry.
func setupRegistryExtensions(registry *FileTypeRegistry) {
registry.AddImageExtension(".png")
registry.AddImageExtension(".jpg")
registry.AddBinaryExtension(".exe")
registry.AddBinaryExtension(".dll")
registry.AddLanguageMapping(".go", "go")
registry.AddLanguageMapping(".py", "python")
}
// verifyExtensionsEnabled verifies that extensions are enabled before disabling.
func verifyExtensionsEnabled(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
if !registry.IsImage(shared.TestFilePNG) {
t.Error("Expected .png to be image before disabling")
}
if !registry.IsBinary(shared.TestFileEXE) {
t.Error("Expected .exe to be binary before disabling")
}
if registry.Language(shared.TestFileGo) != "go" {
t.Error("Expected .go to map to go before disabling")
}
}
// verifyExtensionsDisabled verifies that disabled extensions no longer work.
func verifyExtensionsDisabled(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
if registry.IsImage(shared.TestFilePNG) {
t.Error("Expected .png to not be image after disabling")
}
if registry.IsBinary(shared.TestFileEXE) {
t.Error("Expected .exe to not be binary after disabling")
}
if registry.Language(shared.TestFileGo) != "" {
t.Error("Expected .go to not map to language after disabling")
}
}
// verifyRemainingExtensions verifies that non-disabled extensions still work.
func verifyRemainingExtensions(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
if !registry.IsImage(shared.TestFileJPG) {
t.Error("Expected .jpg to still be image after disabling .png")
}
if !registry.IsBinary(shared.TestFileDLL) {
t.Error("Expected .dll to still be binary after disabling .exe")
}
if registry.Language(shared.TestFilePy) != "python" {
t.Error("Expected .py to still map to python after disabling .go")
}
}
// verifyEmptyValueHandling verifies handling of empty values.
func verifyEmptyValueHandling(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
if registry.IsImage("test") {
t.Error("Expected empty extension to not be added as image")
}
if !registry.IsImage(shared.TestFileValid) {
t.Error("Expected .valid to be added as image")
}
if registry.IsBinary("test") {
t.Error("Expected empty extension to not be added as binary")
}
if !registry.IsBinary(shared.TestFileValid) {
t.Error("Expected .valid to be added as binary")
}
if registry.Language("test") != "" {
t.Error("Expected empty extension to not be added as language")
}
if registry.Language(shared.TestFileValid) != "" {
t.Error("Expected .valid with empty language to not be added")
}
if registry.Language("test.good") != "good" {
t.Error("Expected .good to map to good")
}
}
// verifyCaseInsensitiveHandling verifies case insensitive handling.
func verifyCaseInsensitiveHandling(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
if !registry.IsImage(shared.TestFileWebP) {
t.Error("Expected .webp (lowercase) to work after adding .WEBP")
}
if !registry.IsImage("test.WEBP") {
t.Error("Expected .WEBP (uppercase) to work")
}
if !registry.IsBinary("test.custom") {
t.Error("Expected .custom (lowercase) to work after adding .CUSTOM")
}
if !registry.IsBinary("test.CUSTOM") {
t.Error("Expected .CUSTOM (uppercase) to work")
}
if registry.Language("test.zig") != zigLang {
t.Error("Expected .zig (lowercase) to work after adding .ZIG")
}
if registry.Language("test.ZIG") != zigLang {
t.Error("Expected .ZIG (uppercase) to work")
}
}
// TestConfigureFromSettings tests the global configuration function.
func TestConfigureFromSettings(t *testing.T) {
// Reset registry to ensure clean state
ResetRegistryForTesting()
// Ensure cleanup runs even if test fails
t.Cleanup(ResetRegistryForTesting)
// Test configuration application
customImages := []string{".webp", ".avif"}
customBinary := []string{".custom"}
customLanguages := map[string]string{".zig": "zig"}
customLanguages := map[string]string{".zig": zigLang}
disabledImages := []string{".gif"} // Disable default extension
disabledBinary := []string{".exe"} // Disable default extension
disabledLanguages := []string{".rb"} // Disable default extension
err := ConfigureFromSettings(RegistryConfig{
CustomImages: customImages,
CustomBinary: customBinary,
CustomLanguages: customLanguages,
DisabledImages: disabledImages,
DisabledBinary: disabledBinary,
DisabledLanguages: disabledLanguages,
})
require.NoError(t, err)
ConfigureFromSettings(
customImages,
customBinary,
customLanguages,
disabledImages,
disabledBinary,
disabledLanguages,
)
// Test that custom extensions work
if !IsImage("test.webp") {
if !IsImage(shared.TestFileWebP) {
t.Error("Expected custom image extension .webp to work")
}
if !IsBinary("test.custom") {
t.Error("Expected custom binary extension .custom to work")
}
if GetLanguage("test.zig") != "zig" {
if Language("test.zig") != zigLang {
t.Error("Expected custom language .zig to work")
}
@@ -222,41 +268,43 @@ func TestConfigureFromSettings(t *testing.T) {
if IsImage("test.gif") {
t.Error("Expected disabled image extension .gif to not work")
}
if IsBinary("test.exe") {
if IsBinary(shared.TestFileEXE) {
t.Error("Expected disabled binary extension .exe to not work")
}
if GetLanguage("test.rb") != "" {
if Language("test.rb") != "" {
t.Error("Expected disabled language extension .rb to not work")
}
// Test that non-disabled defaults still work
if !IsImage("test.png") {
if !IsImage(shared.TestFilePNG) {
t.Error("Expected non-disabled image extension .png to still work")
}
if !IsBinary("test.dll") {
if !IsBinary(shared.TestFileDLL) {
t.Error("Expected non-disabled binary extension .dll to still work")
}
if GetLanguage("test.go") != "go" {
if Language(shared.TestFileGo) != "go" {
t.Error("Expected non-disabled language extension .go to still work")
}
// Test multiple calls don't override previous configuration
err = ConfigureFromSettings(RegistryConfig{
CustomImages: []string{".extra"},
CustomBinary: []string{},
CustomLanguages: map[string]string{},
DisabledImages: []string{},
DisabledBinary: []string{},
DisabledLanguages: []string{},
})
require.NoError(t, err)
ConfigureFromSettings(
[]string{".extra"},
[]string{},
map[string]string{},
[]string{},
[]string{},
[]string{},
)
// Previous configuration should still work
if !IsImage("test.webp") {
if !IsImage(shared.TestFileWebP) {
t.Error("Expected previous configuration to persist")
}
// New configuration should also work
if !IsImage("test.extra") {
t.Error("Expected new configuration to be applied")
}
// Reset registry after test to avoid affecting other tests
ResetRegistryForTesting()
}

37
fileproc/filetypes_detection_test.go
View File

@@ -2,31 +2,34 @@ package fileproc
import (
"testing"
"github.com/ivuorinen/gibidify/shared"
)
// newTestRegistry creates a fresh registry instance for testing to avoid global state pollution.
func newTestRegistry() *FileTypeRegistry {
// createTestRegistry creates a fresh FileTypeRegistry instance for testing.
// This helper reduces code duplication and ensures consistent registry initialization.
func createTestRegistry() *FileTypeRegistry {
return &FileTypeRegistry{
imageExts: getImageExtensions(),
binaryExts: getBinaryExtensions(),
languageMap: getLanguageMap(),
extCache: make(map[string]string, 1000),
resultCache: make(map[string]FileTypeResult, 500),
maxCacheSize: 500,
extCache: make(map[string]string, shared.FileTypeRegistryMaxCacheSize),
resultCache: make(map[string]FileTypeResult, shared.FileTypeRegistryMaxCacheSize),
maxCacheSize: shared.FileTypeRegistryMaxCacheSize,
}
}
// TestFileTypeRegistry_LanguageDetection tests the language detection functionality.
func TestFileTypeRegistry_LanguageDetection(t *testing.T) {
registry := newTestRegistry()
func TestFileTypeRegistryLanguageDetection(t *testing.T) {
registry := createTestRegistry()
tests := []struct {
filename string
expected string
}{
// Programming languages
{"main.go", "go"},
{"script.py", "python"},
{shared.TestFileMainGo, "go"},
{shared.TestFileScriptPy, "python"},
{"app.js", "javascript"},
{"component.tsx", "typescript"},
{"service.ts", "typescript"},
@@ -96,17 +99,17 @@ func TestFileTypeRegistry_LanguageDetection(t *testing.T) {
for _, tt := range tests {
t.Run(tt.filename, func(t *testing.T) {
result := registry.GetLanguage(tt.filename)
result := registry.Language(tt.filename)
if result != tt.expected {
t.Errorf("GetLanguage(%q) = %q, expected %q", tt.filename, result, tt.expected)
t.Errorf("Language(%q) = %q, expected %q", tt.filename, result, tt.expected)
}
})
}
}
// TestFileTypeRegistry_ImageDetection tests the image detection functionality.
func TestFileTypeRegistry_ImageDetection(t *testing.T) {
registry := newTestRegistry()
func TestFileTypeRegistryImageDetection(t *testing.T) {
registry := createTestRegistry()
tests := []struct {
filename string
@@ -114,7 +117,7 @@ func TestFileTypeRegistry_ImageDetection(t *testing.T) {
}{
// Common image formats
{"photo.png", true},
{"image.jpg", true},
{shared.TestFileImageJPG, true},
{"picture.jpeg", true},
{"animation.gif", true},
{"bitmap.bmp", true},
@@ -155,8 +158,8 @@ func TestFileTypeRegistry_ImageDetection(t *testing.T) {
}
// TestFileTypeRegistry_BinaryDetection tests the binary detection functionality.
func TestFileTypeRegistry_BinaryDetection(t *testing.T) {
registry := newTestRegistry()
func TestFileTypeRegistryBinaryDetection(t *testing.T) {
registry := createTestRegistry()
tests := []struct {
filename string
@@ -214,7 +217,7 @@ func TestFileTypeRegistry_BinaryDetection(t *testing.T) {
// Non-binary files
{"document.txt", false},
{"script.py", false},
{shared.TestFileScriptPy, false},
{"config.json", false},
{"style.css", false},
{"page.html", false},

52
fileproc/filetypes_edge_cases_test.go
View File

@@ -2,11 +2,13 @@ package fileproc
import (
"testing"
"github.com/ivuorinen/gibidify/shared"
)
// TestFileTypeRegistry_EdgeCases tests edge cases and boundary conditions.
func TestFileTypeRegistry_EdgeCases(t *testing.T) {
registry := GetDefaultRegistry()
func TestFileTypeRegistryEdgeCases(t *testing.T) {
registry := DefaultRegistry()
// Test various edge cases for filename handling
edgeCases := []struct {
@@ -35,19 +37,19 @@ func TestFileTypeRegistry_EdgeCases(t *testing.T) {
// These should not panic
_ = registry.IsImage(tc.filename)
_ = registry.IsBinary(tc.filename)
_ = registry.GetLanguage(tc.filename)
_ = registry.Language(tc.filename)
// Global functions should also not panic
_ = IsImage(tc.filename)
_ = IsBinary(tc.filename)
_ = GetLanguage(tc.filename)
_ = Language(tc.filename)
})
}
}
// TestFileTypeRegistry_MinimumExtensionLength tests the minimum extension length requirement.
func TestFileTypeRegistry_MinimumExtensionLength(t *testing.T) {
registry := GetDefaultRegistry()
func TestFileTypeRegistryMinimumExtensionLength(t *testing.T) {
registry := DefaultRegistry()
tests := []struct {
filename string
@@ -65,18 +67,18 @@ func TestFileTypeRegistry_MinimumExtensionLength(t *testing.T) {
for _, tt := range tests {
t.Run(tt.filename, func(t *testing.T) {
result := registry.GetLanguage(tt.filename)
result := registry.Language(tt.filename)
if result != tt.expected {
t.Errorf("GetLanguage(%q) = %q, expected %q", tt.filename, result, tt.expected)
t.Errorf("Language(%q) = %q, expected %q", tt.filename, result, tt.expected)
}
})
}
}
// Benchmark tests for performance validation
func BenchmarkFileTypeRegistry_IsImage(b *testing.B) {
registry := GetDefaultRegistry()
filename := "test.png"
// Benchmark tests for performance validation.
func BenchmarkFileTypeRegistryIsImage(b *testing.B) {
registry := DefaultRegistry()
filename := shared.TestFilePNG
b.ResetTimer()
for i := 0; i < b.N; i++ {
@@ -84,9 +86,9 @@ func BenchmarkFileTypeRegistry_IsImage(b *testing.B) {
}
}
func BenchmarkFileTypeRegistry_IsBinary(b *testing.B) {
registry := GetDefaultRegistry()
filename := "test.exe"
func BenchmarkFileTypeRegistryIsBinary(b *testing.B) {
registry := DefaultRegistry()
filename := shared.TestFileEXE
b.ResetTimer()
for i := 0; i < b.N; i++ {
@@ -94,35 +96,35 @@ func BenchmarkFileTypeRegistry_IsBinary(b *testing.B) {
}
}
func BenchmarkFileTypeRegistry_GetLanguage(b *testing.B) {
registry := GetDefaultRegistry()
filename := "test.go"
func BenchmarkFileTypeRegistryLanguage(b *testing.B) {
registry := DefaultRegistry()
filename := shared.TestFileGo
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = registry.GetLanguage(filename)
_ = registry.Language(filename)
}
}
func BenchmarkFileTypeRegistry_GlobalFunctions(b *testing.B) {
filename := "test.go"
func BenchmarkFileTypeRegistryGlobalFunctions(b *testing.B) {
filename := shared.TestFileGo
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = IsImage(filename)
_ = IsBinary(filename)
_ = GetLanguage(filename)
_ = Language(filename)
}
}
func BenchmarkFileTypeRegistry_ConcurrentAccess(b *testing.B) {
filename := "test.go"
func BenchmarkFileTypeRegistryConcurrentAccess(b *testing.B) {
filename := shared.TestFileGo
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = IsImage(filename)
_ = IsBinary(filename)
_ = GetLanguage(filename)
_ = Language(filename)
}
})
}

340
fileproc/filetypes_registry_test.go
View File

@@ -2,136 +2,254 @@ package fileproc
import (
"testing"
"github.com/ivuorinen/gibidify/shared"
)
// TestFileTypeRegistry_ModificationMethods tests the modification methods of FileTypeRegistry.
func TestFileTypeRegistry_ModificationMethods(t *testing.T) {
// Create a new registry instance for testing
registry := &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
}
// TestFileTypeRegistryAddImageExtension tests adding image extensions.
func TestFileTypeRegistryAddImageExtension(t *testing.T) {
registry := createModificationTestRegistry()
// Test AddImageExtension
t.Run("AddImageExtension", func(t *testing.T) {
// Add a new image extension
registry.AddImageExtension(".webp")
if !registry.IsImage("test.webp") {
t.Errorf("Expected .webp to be recognized as image after adding")
}
// Test case-insensitive addition
registry.AddImageExtension(".AVIF")
if !registry.IsImage("test.avif") {
t.Errorf("Expected .avif to be recognized as image after adding .AVIF")
}
if !registry.IsImage("test.AVIF") {
t.Errorf("Expected .AVIF to be recognized as image")
}
// Test with dot prefix
registry.AddImageExtension("heic")
if registry.IsImage("test.heic") {
t.Errorf("Expected extension without dot to not work")
}
// Test with proper dot prefix
registry.AddImageExtension(".heic")
if !registry.IsImage("test.heic") {
t.Errorf("Expected .heic to be recognized as image")
}
})
// Test AddBinaryExtension
t.Run("AddBinaryExtension", func(t *testing.T) {
// Add a new binary extension
registry.AddBinaryExtension(".custom")
if !registry.IsBinary("file.custom") {
t.Errorf("Expected .custom to be recognized as binary after adding")
}
// Test case-insensitive addition
registry.AddBinaryExtension(".SPECIAL")
if !registry.IsBinary("file.special") {
t.Errorf("Expected .special to be recognized as binary after adding .SPECIAL")
}
if !registry.IsBinary("file.SPECIAL") {
t.Errorf("Expected .SPECIAL to be recognized as binary")
}
// Test with dot prefix
registry.AddBinaryExtension("bin")
if registry.IsBinary("file.bin") {
t.Errorf("Expected extension without dot to not work")
}
// Test with proper dot prefix
registry.AddBinaryExtension(".bin")
if !registry.IsBinary("file.bin") {
t.Errorf("Expected .bin to be recognized as binary")
}
})
// Test AddLanguageMapping
t.Run("AddLanguageMapping", func(t *testing.T) {
// Add a new language mapping
registry.AddLanguageMapping(".xyz", "CustomLang")
if lang := registry.GetLanguage("file.xyz"); lang != "CustomLang" {
t.Errorf("Expected CustomLang, got %s", lang)
}
// Test case-insensitive addition
registry.AddLanguageMapping(".ABC", "UpperLang")
if lang := registry.GetLanguage("file.abc"); lang != "UpperLang" {
t.Errorf("Expected UpperLang, got %s", lang)
}
if lang := registry.GetLanguage("file.ABC"); lang != "UpperLang" {
t.Errorf("Expected UpperLang for uppercase, got %s", lang)
}
// Test with dot prefix
registry.AddLanguageMapping("nolang", "NoLang")
if lang := registry.GetLanguage("file.nolang"); lang == "NoLang" {
t.Errorf("Expected extension without dot to not work")
}
// Test with proper dot prefix
registry.AddLanguageMapping(".nolang", "NoLang")
if lang := registry.GetLanguage("file.nolang"); lang != "NoLang" {
t.Errorf("Expected NoLang, got %s", lang)
}
// Test overriding existing mapping
registry.AddLanguageMapping(".xyz", "NewCustomLang")
if lang := registry.GetLanguage("file.xyz"); lang != "NewCustomLang" {
t.Errorf("Expected NewCustomLang after override, got %s", lang)
}
})
testImageExtensionModifications(t, registry)
}
// TestFileTypeRegistry_DefaultRegistryConsistency tests default registry behavior.
func TestFileTypeRegistry_DefaultRegistryConsistency(t *testing.T) {
registry := GetDefaultRegistry()
// TestFileTypeRegistryAddBinaryExtension tests adding binary extensions.
func TestFileTypeRegistryAddBinaryExtension(t *testing.T) {
registry := createModificationTestRegistry()
testBinaryExtensionModifications(t, registry)
}
// TestFileTypeRegistryAddLanguageMapping tests adding language mappings.
func TestFileTypeRegistryAddLanguageMapping(t *testing.T) {
registry := createModificationTestRegistry()
testLanguageMappingModifications(t, registry)
}
// createModificationTestRegistry creates a registry for modification tests.
func createModificationTestRegistry() *FileTypeRegistry {
return &FileTypeRegistry{
imageExts: make(map[string]bool),
binaryExts: make(map[string]bool),
languageMap: make(map[string]string),
extCache: make(map[string]string, shared.FileTypeRegistryMaxCacheSize),
resultCache: make(map[string]FileTypeResult, shared.FileTypeRegistryMaxCacheSize),
maxCacheSize: shared.FileTypeRegistryMaxCacheSize,
}
}
// testImageExtensionModifications tests image extension modifications.
func testImageExtensionModifications(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
// Add a new image extension
registry.AddImageExtension(".webp")
verifyImageExtension(t, registry, ".webp", shared.TestFileWebP, true)
// Test case-insensitive addition
registry.AddImageExtension(".AVIF")
verifyImageExtension(t, registry, ".AVIF", "test.avif", true)
verifyImageExtension(t, registry, ".AVIF", "test.AVIF", true)
// Test with dot prefix
registry.AddImageExtension("heic")
verifyImageExtension(t, registry, "heic", "test.heic", false)
// Test with proper dot prefix
registry.AddImageExtension(".heic")
verifyImageExtension(t, registry, ".heic", "test.heic", true)
}
// testBinaryExtensionModifications tests binary extension modifications.
func testBinaryExtensionModifications(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
// Add a new binary extension
registry.AddBinaryExtension(".custom")
verifyBinaryExtension(t, registry, ".custom", "file.custom", true)
// Test case-insensitive addition
registry.AddBinaryExtension(shared.TestExtensionSpecial)
verifyBinaryExtension(t, registry, shared.TestExtensionSpecial, "file.special", true)
verifyBinaryExtension(t, registry, shared.TestExtensionSpecial, "file.SPECIAL", true)
// Test with dot prefix
registry.AddBinaryExtension("bin")
verifyBinaryExtension(t, registry, "bin", "file.bin", false)
// Test with proper dot prefix
registry.AddBinaryExtension(".bin")
verifyBinaryExtension(t, registry, ".bin", "file.bin", true)
}
// testLanguageMappingModifications tests language mapping modifications.
func testLanguageMappingModifications(t *testing.T, registry *FileTypeRegistry) {
t.Helper()
// Add a new language mapping
registry.AddLanguageMapping(".xyz", "CustomLang")
verifyLanguageMapping(t, registry, "file.xyz", "CustomLang")
// Test case-insensitive addition
registry.AddLanguageMapping(".ABC", "UpperLang")
verifyLanguageMapping(t, registry, "file.abc", "UpperLang")
verifyLanguageMapping(t, registry, "file.ABC", "UpperLang")
// Test with dot prefix (should not work)
registry.AddLanguageMapping("nolang", "NoLang")
verifyLanguageMappingAbsent(t, registry, "nolang", "file.nolang")
// Test with proper dot prefix
registry.AddLanguageMapping(".nolang", "NoLang")
verifyLanguageMapping(t, registry, "file.nolang", "NoLang")
// Test overriding existing mapping
registry.AddLanguageMapping(".xyz", "NewCustomLang")
verifyLanguageMapping(t, registry, "file.xyz", "NewCustomLang")
}
// verifyImageExtension verifies image extension behavior.
func verifyImageExtension(t *testing.T, registry *FileTypeRegistry, ext, filename string, expected bool) {
t.Helper()
if registry.IsImage(filename) != expected {
if expected {
t.Errorf("Expected %s to be recognized as image after adding %s", filename, ext)
} else {
t.Errorf(shared.TestMsgExpectedExtensionWithoutDot)
}
}
}
// verifyBinaryExtension verifies binary extension behavior.
func verifyBinaryExtension(t *testing.T, registry *FileTypeRegistry, ext, filename string, expected bool) {
t.Helper()
if registry.IsBinary(filename) != expected {
if expected {
t.Errorf("Expected %s to be recognized as binary after adding %s", filename, ext)
} else {
t.Errorf(shared.TestMsgExpectedExtensionWithoutDot)
}
}
}
// verifyLanguageMapping verifies language mapping behavior.
func verifyLanguageMapping(t *testing.T, registry *FileTypeRegistry, filename, expectedLang string) {
t.Helper()
lang := registry.Language(filename)
if lang != expectedLang {
t.Errorf("Expected %s, got %s for %s", expectedLang, lang, filename)
}
}
// verifyLanguageMappingAbsent verifies that a language mapping is absent.
func verifyLanguageMappingAbsent(t *testing.T, registry *FileTypeRegistry, _ string, filename string) {
t.Helper()
lang := registry.Language(filename)
if lang != "" {
t.Errorf(shared.TestMsgExpectedExtensionWithoutDot+", but got %s", lang)
}
}
// TestFileTypeRegistryDefaultRegistryConsistency tests default registry behavior.
func TestFileTypeRegistryDefaultRegistryConsistency(t *testing.T) {
registry := DefaultRegistry()
// Test that registry methods work consistently
if !registry.IsImage("test.png") {
if !registry.IsImage(shared.TestFilePNG) {
t.Error("Expected .png to be recognized as image")
}
if !registry.IsBinary("test.exe") {
if !registry.IsBinary(shared.TestFileEXE) {
t.Error("Expected .exe to be recognized as binary")
}
if lang := registry.GetLanguage("test.go"); lang != "go" {
if lang := registry.Language(shared.TestFileGo); lang != "go" {
t.Errorf("Expected go, got %s", lang)
}
// Test that multiple calls return consistent results
for i := 0; i < 5; i++ {
if !registry.IsImage("test.jpg") {
if !registry.IsImage(shared.TestFileJPG) {
t.Errorf("Iteration %d: Expected .jpg to be recognized as image", i)
}
if registry.IsBinary("test.txt") {
if registry.IsBinary(shared.TestFileTXT) {
t.Errorf("Iteration %d: Expected .txt to not be recognized as binary", i)
}
}
}
// TestFileTypeRegistryGetStats tests the GetStats method.
func TestFileTypeRegistryGetStats(t *testing.T) {
// Ensure clean, isolated state
ResetRegistryForTesting()
t.Cleanup(ResetRegistryForTesting)
registry := DefaultRegistry()
// Call some methods to populate cache and update stats
registry.IsImage(shared.TestFilePNG)
registry.IsBinary(shared.TestFileEXE)
registry.Language(shared.TestFileGo)
// Repeat to generate cache hits
registry.IsImage(shared.TestFilePNG)
registry.IsBinary(shared.TestFileEXE)
registry.Language(shared.TestFileGo)
// Get stats
stats := registry.Stats()
// Verify stats structure - all values are uint64 and therefore non-negative by definition
// We can verify they exist and are properly initialized
// Test that stats include our calls
if stats.TotalLookups < 6 { // We made at least 6 calls above
t.Errorf("Expected at least 6 total lookups, got %d", stats.TotalLookups)
}
// Total lookups should equal hits + misses
if stats.TotalLookups != stats.CacheHits+stats.CacheMisses {
t.Errorf("Total lookups (%d) should equal hits (%d) + misses (%d)",
stats.TotalLookups, stats.CacheHits, stats.CacheMisses)
}
// With repeated lookups we should see some cache hits
if stats.CacheHits == 0 {
t.Error("Expected some cache hits after repeated lookups")
}
}
// TestFileTypeRegistryGetCacheInfo tests the GetCacheInfo method.
func TestFileTypeRegistryGetCacheInfo(t *testing.T) {
// Ensure clean, isolated state
ResetRegistryForTesting()
t.Cleanup(ResetRegistryForTesting)
registry := DefaultRegistry()
// Call some methods to populate cache
registry.IsImage("test1.png")
registry.IsBinary("test2.exe")
registry.Language("test3.go")
registry.IsImage("test4.jpg")
registry.IsBinary("test5.dll")
// Get cache info
extCacheSize, resultCacheSize, maxCacheSize := registry.CacheInfo()
// Verify cache info
if extCacheSize < 0 {
t.Error("Expected non-negative extension cache size")
}
if resultCacheSize < 0 {
t.Error("Expected non-negative result cache size")
}
if maxCacheSize <= 0 {
t.Error("Expected positive max cache size")
}
// We should have some cache entries from our calls
totalCacheSize := extCacheSize + resultCacheSize
if totalCacheSize == 0 {
t.Error("Expected some cache entries after multiple calls")
}
}

6
fileproc/formats.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
// FileData represents a single file's path and content.
@@ -23,6 +24,7 @@ type FormatWriter interface {
// detectLanguage tries to infer the code block language from the file extension.
func detectLanguage(filePath string) string {
registry := GetDefaultRegistry()
return registry.GetLanguage(filePath)
registry := DefaultRegistry()
return registry.Language(filePath)
}

4
fileproc/ignore_rules.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
@@ -34,6 +35,7 @@ func loadIgnoreRules(currentDir string, parentRules []ignoreRule) []ignoreRule {
func tryLoadIgnoreFile(dir, fileName string) *ignoreRule {
ignorePath := filepath.Join(dir, fileName)
if info, err := os.Stat(ignorePath); err == nil && !info.IsDir() {
//nolint:errcheck // Regex compile error handled by validation, safe to ignore here
if gi, err := ignore.CompileIgnoreFile(ignorePath); err == nil {
return &ignoreRule{
base: dir,
@@ -41,6 +43,7 @@ func tryLoadIgnoreFile(dir, fileName string) *ignoreRule {
}
}
}
return nil
}
@@ -51,6 +54,7 @@ func matchesIgnoreRules(fullPath string, rules []ignoreRule) bool {
return true
}
}
return false
}

115
fileproc/json_writer.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
@@ -6,7 +7,7 @@ import (
"io"
"os"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// JSONWriter handles JSON format output with streaming support.
@@ -27,42 +28,27 @@ func NewJSONWriter(outFile *os.File) *JSONWriter {
func (w *JSONWriter) Start(prefix, suffix string) error {
// Start JSON structure
if _, err := w.outFile.WriteString(`{"prefix":"`); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write JSON start",
)
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write JSON start")
}
// Write escaped prefix
escapedPrefix := gibidiutils.EscapeForJSON(prefix)
if err := gibidiutils.WriteWithErrorWrap(w.outFile, escapedPrefix, "failed to write JSON prefix", ""); err != nil {
return err
escapedPrefix := shared.EscapeForJSON(prefix)
if err := shared.WriteWithErrorWrap(w.outFile, escapedPrefix, "failed to write JSON prefix", ""); err != nil {
return fmt.Errorf("writing JSON prefix: %w", err)
}
if _, err := w.outFile.WriteString(`","suffix":"`); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write JSON middle",
)
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write JSON middle")
}
// Write escaped suffix
escapedSuffix := gibidiutils.EscapeForJSON(suffix)
if err := gibidiutils.WriteWithErrorWrap(w.outFile, escapedSuffix, "failed to write JSON suffix", ""); err != nil {
return err
escapedSuffix := shared.EscapeForJSON(suffix)
if err := shared.WriteWithErrorWrap(w.outFile, escapedSuffix, "failed to write JSON suffix", ""); err != nil {
return fmt.Errorf("writing JSON suffix: %w", err)
}
if _, err := w.outFile.WriteString(`","files":[`); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write JSON files start",
)
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write JSON files start")
}
return nil
@@ -72,12 +58,7 @@ func (w *JSONWriter) Start(prefix, suffix string) error {
func (w *JSONWriter) WriteFile(req WriteRequest) error {
if !w.firstFile {
if _, err := w.outFile.WriteString(","); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write JSON separator",
)
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write JSON separator")
}
}
w.firstFile = false
@@ -85,6 +66,7 @@ func (w *JSONWriter) WriteFile(req WriteRequest) error {
if req.IsStream {
return w.writeStreaming(req)
}
return w.writeInline(req)
}
@@ -92,22 +74,25 @@ func (w *JSONWriter) WriteFile(req WriteRequest) error {
func (w *JSONWriter) Close() error {
// Close JSON structure
if _, err := w.outFile.WriteString("]}"); err != nil {
return gibidiutils.WrapError(err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, "failed to write JSON end")
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write JSON end")
}
return nil
}
// writeStreaming writes a large file as JSON in streaming chunks.
func (w *JSONWriter) writeStreaming(req WriteRequest) error {
defer gibidiutils.SafeCloseReader(req.Reader, req.Path)
defer shared.SafeCloseReader(req.Reader, req.Path)
language := detectLanguage(req.Path)
// Write file start
escapedPath := gibidiutils.EscapeForJSON(req.Path)
escapedPath := shared.EscapeForJSON(req.Path)
if _, err := fmt.Fprintf(w.outFile, `{"path":"%s","language":"%s","content":"`, escapedPath, language); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write JSON file start",
).WithFilePath(req.Path)
}
@@ -119,8 +104,10 @@ func (w *JSONWriter) writeStreaming(req WriteRequest) error {
// Write file end
if _, err := w.outFile.WriteString(`"}`); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write JSON file end",
).WithFilePath(req.Path)
}
@@ -139,50 +126,44 @@ func (w *JSONWriter) writeInline(req WriteRequest) error {
encoded, err := json.Marshal(fileData)
if err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingEncode,
return shared.WrapError(
err,
shared.ErrorTypeProcessing,
shared.CodeProcessingEncode,
"failed to marshal JSON",
).WithFilePath(req.Path)
}
if _, err := w.outFile.Write(encoded); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write JSON file",
).WithFilePath(req.Path)
}
return nil
}
// streamJSONContent streams content with JSON escaping.
func (w *JSONWriter) streamJSONContent(reader io.Reader, path string) error {
return gibidiutils.StreamContent(reader, w.outFile, StreamChunkSize, path, func(chunk []byte) []byte {
escaped := gibidiutils.EscapeForJSON(string(chunk))
return []byte(escaped)
})
if err := shared.StreamContent(
reader, w.outFile, shared.FileProcessingStreamChunkSize, path, func(chunk []byte) []byte {
escaped := shared.EscapeForJSON(string(chunk))
return []byte(escaped)
},
); err != nil {
return fmt.Errorf("streaming JSON content: %w", err)
}
return nil
}
// startJSONWriter handles JSON format output with streaming support.
func startJSONWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
defer close(done)
writer := NewJSONWriter(outFile)
// Start writing
if err := writer.Start(prefix, suffix); err != nil {
gibidiutils.LogError("Failed to write JSON start", err)
return
}
// Process files
for req := range writeCh {
if err := writer.WriteFile(req); err != nil {
gibidiutils.LogError("Failed to write JSON file", err)
}
}
// Close writer
if err := writer.Close(); err != nil {
gibidiutils.LogError("Failed to write JSON end", err)
}
startFormatWriter(outFile, writeCh, done, prefix, suffix, func(f *os.File) FormatWriter {
return NewJSONWriter(f)
})
}

179
fileproc/markdown_writer.go
View File

@@ -1,18 +1,17 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// MarkdownWriter handles Markdown format output with streaming support.
type MarkdownWriter struct {
outFile *os.File
suffix string
}
// NewMarkdownWriter creates a new markdown writer.
@@ -20,18 +19,17 @@ func NewMarkdownWriter(outFile *os.File) *MarkdownWriter {
return &MarkdownWriter{outFile: outFile}
}
// Start writes the markdown header.
func (w *MarkdownWriter) Start(prefix, _ string) error {
// Start writes the markdown header and stores the suffix for later use.
func (w *MarkdownWriter) Start(prefix, suffix string) error {
// Store suffix for use in Close method
w.suffix = suffix
if prefix != "" {
if _, err := fmt.Fprintf(w.outFile, "# %s\n\n", prefix); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write prefix",
)
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write prefix")
}
}
return nil
}
@@ -40,71 +38,15 @@ func (w *MarkdownWriter) WriteFile(req WriteRequest) error {
if req.IsStream {
return w.writeStreaming(req)
}
return w.writeInline(req)
}
// Close writes the markdown footer.
func (w *MarkdownWriter) Close(suffix string) error {
if suffix != "" {
if _, err := fmt.Fprintf(w.outFile, "\n# %s\n", suffix); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write suffix",
)
}
}
return nil
}
// validateMarkdownPath validates a file path for markdown output.
func validateMarkdownPath(path string) error {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationRequired,
"file path cannot be empty",
"",
nil,
)
}
// Reject absolute paths
if filepath.IsAbs(trimmed) {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"absolute paths are not allowed",
trimmed,
map[string]any{"path": trimmed},
)
}
// Clean and validate path components
cleaned := filepath.Clean(trimmed)
if filepath.IsAbs(cleaned) || strings.HasPrefix(cleaned, "/") {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path must be relative",
trimmed,
map[string]any{"path": trimmed, "cleaned": cleaned},
)
}
// Check for path traversal in components
components := strings.Split(filepath.ToSlash(cleaned), "/")
for _, component := range components {
if component == ".." {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path traversal not allowed",
trimmed,
map[string]any{"path": trimmed, "cleaned": cleaned},
)
// Close writes the markdown footer using the suffix stored in Start.
func (w *MarkdownWriter) Close() error {
if w.suffix != "" {
if _, err := fmt.Fprintf(w.outFile, "\n# %s\n", w.suffix); err != nil {
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write suffix")
}
}
@@ -113,44 +55,32 @@ func validateMarkdownPath(path string) error {
// writeStreaming writes a large file in streaming chunks.
func (w *MarkdownWriter) writeStreaming(req WriteRequest) error {
// Validate path before use
if err := validateMarkdownPath(req.Path); err != nil {
return err
}
// Check for nil reader
if req.Reader == nil {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationRequired,
"nil reader in write request",
"",
nil,
).WithFilePath(req.Path)
}
defer gibidiutils.SafeCloseReader(req.Reader, req.Path)
defer shared.SafeCloseReader(req.Reader, req.Path)
language := detectLanguage(req.Path)
// Write file header
safePath := gibidiutils.EscapeForMarkdown(req.Path)
if _, err := fmt.Fprintf(w.outFile, "## File: `%s`\n```%s\n", safePath, language); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
if _, err := fmt.Fprintf(w.outFile, "## File: `%s`\n```%s\n", req.Path, language); err != nil {
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write file header",
).WithFilePath(req.Path)
}
// Stream file content in chunks
if err := w.streamContent(req.Reader, req.Path); err != nil {
return err
chunkSize := shared.FileProcessingStreamChunkSize
if err := shared.StreamContent(req.Reader, w.outFile, chunkSize, req.Path, nil); err != nil {
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "streaming content for markdown file")
}
// Write file footer
if _, err := w.outFile.WriteString("\n```\n\n"); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write file footer",
).WithFilePath(req.Path)
}
@@ -160,55 +90,24 @@ func (w *MarkdownWriter) writeStreaming(req WriteRequest) error {
// writeInline writes a small file directly from content.
func (w *MarkdownWriter) writeInline(req WriteRequest) error {
// Validate path before use
if err := validateMarkdownPath(req.Path); err != nil {
return err
}
language := detectLanguage(req.Path)
safePath := gibidiutils.EscapeForMarkdown(req.Path)
formatted := fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", safePath, language, req.Content)
formatted := fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", req.Path, language, req.Content)
if _, err := w.outFile.WriteString(formatted); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write inline content",
).WithFilePath(req.Path)
}
return nil
}
// streamContent streams file content in chunks.
func (w *MarkdownWriter) streamContent(reader io.Reader, path string) error {
return gibidiutils.StreamContent(reader, w.outFile, StreamChunkSize, path, nil)
}
// startMarkdownWriter handles Markdown format output with streaming support.
func startMarkdownWriter(
outFile *os.File,
writeCh <-chan WriteRequest,
done chan<- struct{},
prefix, suffix string,
) {
defer close(done)
writer := NewMarkdownWriter(outFile)
// Start writing
if err := writer.Start(prefix, suffix); err != nil {
gibidiutils.LogError("Failed to write markdown prefix", err)
return
}
// Process files
for req := range writeCh {
if err := writer.WriteFile(req); err != nil {
gibidiutils.LogError("Failed to write markdown file", err)
}
}
// Close writer
if err := writer.Close(suffix); err != nil {
gibidiutils.LogError("Failed to write markdown suffix", err)
}
func startMarkdownWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
startFormatWriter(outFile, writeCh, done, prefix, suffix, func(f *os.File) FormatWriter {
return NewMarkdownWriter(f)
})
}

407
fileproc/processor.go
View File

@@ -9,21 +9,11 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/gibidiutils"
)
const (
// StreamChunkSize is the size of chunks when streaming large files (64KB).
StreamChunkSize = 65536
// StreamThreshold is the file size above which we use streaming (1MB).
StreamThreshold = 1048576
// MaxMemoryBuffer is the maximum memory to use for buffering content (10MB).
MaxMemoryBuffer = 10485760
"github.com/ivuorinen/gibidify/shared"
)
// WriteRequest represents the content to be written.
@@ -32,26 +22,7 @@ type WriteRequest struct {
Content string
IsStream bool
Reader io.Reader
}
// multiReaderCloser wraps an io.Reader with a Close method that closes underlying closers.
type multiReaderCloser struct {
reader io.Reader
closers []io.Closer
}
func (m *multiReaderCloser) Read(p []byte) (n int, err error) {
return m.reader.Read(p)
}
func (m *multiReaderCloser) Close() error {
var firstErr error
for _, c := range m.closers {
if err := c.Close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
Size int64 // File size for streaming files
}
// FileProcessor handles file processing operations.
@@ -65,7 +36,7 @@ type FileProcessor struct {
func NewFileProcessor(rootPath string) *FileProcessor {
return &FileProcessor{
rootPath: rootPath,
sizeLimit: config.GetFileSizeLimit(),
sizeLimit: config.FileSizeLimit(),
resourceMonitor: NewResourceMonitor(),
}
}
@@ -74,45 +45,19 @@ func NewFileProcessor(rootPath string) *FileProcessor {
func NewFileProcessorWithMonitor(rootPath string, monitor *ResourceMonitor) *FileProcessor {
return &FileProcessor{
rootPath: rootPath,
sizeLimit: config.GetFileSizeLimit(),
sizeLimit: config.FileSizeLimit(),
resourceMonitor: monitor,
}
}
// checkContextCancellation checks if context is cancelled and logs an error if so.
// Returns true if context is cancelled, false otherwise.
func (p *FileProcessor) checkContextCancellation(ctx context.Context, filePath, stage string) bool {
select {
case <-ctx.Done():
// Format stage with leading space if provided
stageMsg := stage
if stage != "" {
stageMsg = " " + stage
}
gibidiutils.LogErrorf(
gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitTimeout,
fmt.Sprintf("file processing cancelled%s", stageMsg),
filePath,
nil,
),
"File processing cancelled%s: %s",
stageMsg,
filePath,
)
return true
default:
return false
}
}
// ProcessFile reads the file at filePath and sends a formatted output to outCh.
// It automatically chooses between loading the entire file or streaming based on file size.
func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) {
processor := NewFileProcessor(rootPath)
ctx := context.Background()
processor.ProcessWithContext(ctx, filePath, outCh)
if err := processor.ProcessWithContext(ctx, filePath, outCh); err != nil {
shared.LogErrorf(err, shared.FileProcessingMsgFailedToProcess, filePath)
}
}
// ProcessFileWithMonitor processes a file using a shared resource monitor.
@@ -122,19 +67,25 @@ func ProcessFileWithMonitor(
outCh chan<- WriteRequest,
rootPath string,
monitor *ResourceMonitor,
) {
) error {
if monitor == nil {
monitor = NewResourceMonitor()
}
processor := NewFileProcessorWithMonitor(rootPath, monitor)
processor.ProcessWithContext(ctx, filePath, outCh)
return processor.ProcessWithContext(ctx, filePath, outCh)
}
// Process handles file processing with the configured settings.
func (p *FileProcessor) Process(filePath string, outCh chan<- WriteRequest) {
ctx := context.Background()
p.ProcessWithContext(ctx, filePath, outCh)
if err := p.ProcessWithContext(ctx, filePath, outCh); err != nil {
shared.LogErrorf(err, shared.FileProcessingMsgFailedToProcess, filePath)
}
}
// ProcessWithContext handles file processing with context and resource monitoring.
func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, outCh chan<- WriteRequest) {
func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, outCh chan<- WriteRequest) error {
// Create file processing context with timeout
fileCtx, fileCancel := p.resourceMonitor.CreateFileProcessingContext(ctx)
defer fileCancel()
@@ -142,50 +93,51 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string,
// Wait for rate limiting
if err := p.resourceMonitor.WaitForRateLimit(fileCtx); err != nil {
if errors.Is(err, context.DeadlineExceeded) {
gibidiutils.LogErrorf(
gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitTimeout,
"file processing timeout during rate limiting",
filePath,
nil,
),
"File processing timeout during rate limiting: %s",
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"file processing timeout during rate limiting",
filePath,
nil,
)
shared.LogErrorf(structErr, "File processing timeout during rate limiting: %s", filePath)
return structErr
}
return
return err
}
// Validate file and check resource limits
fileInfo, err := p.validateFileWithLimits(fileCtx, filePath)
if err != nil {
return // Error already logged
return err // Error already logged
}
// Acquire read slot for concurrent processing
if err := p.resourceMonitor.AcquireReadSlot(fileCtx); err != nil {
if errors.Is(err, context.DeadlineExceeded) {
gibidiutils.LogErrorf(
gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitTimeout,
"file processing timeout waiting for read slot",
filePath,
nil,
),
"File processing timeout waiting for read slot: %s",
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"file processing timeout waiting for read slot",
filePath,
nil,
)
shared.LogErrorf(structErr, "File processing timeout waiting for read slot: %s", filePath)
return structErr
}
return
return err
}
defer p.resourceMonitor.ReleaseReadSlot()
// Check hard memory limits before processing
if err := p.resourceMonitor.CheckHardMemoryLimit(); err != nil {
gibidiutils.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath)
return
shared.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath)
return err
}
// Get relative path
@@ -193,61 +145,69 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string,
// Process file with timeout
processStart := time.Now()
defer func() {
// Record successful processing
p.resourceMonitor.RecordFileProcessed(fileInfo.Size())
logrus.Debugf("File processed in %v: %s", time.Since(processStart), filePath)
}()
// Choose processing strategy based on file size
if fileInfo.Size() <= StreamThreshold {
p.processInMemoryWithContext(fileCtx, filePath, relPath, outCh)
if fileInfo.Size() <= shared.FileProcessingStreamThreshold {
err = p.processInMemoryWithContext(fileCtx, filePath, relPath, outCh)
} else {
p.processStreamingWithContext(fileCtx, filePath, relPath, outCh)
err = p.processStreamingWithContext(fileCtx, filePath, relPath, outCh, fileInfo.Size())
}
// Only record success if processing completed without error
if err != nil {
return err
}
// Record successful processing only on success path
p.resourceMonitor.RecordFileProcessed(fileInfo.Size())
logger := shared.GetLogger()
logger.Debugf("File processed in %v: %s", time.Since(processStart), filePath)
return nil
}
// validateFileWithLimits checks if the file can be processed with resource limits.
func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath string) (os.FileInfo, error) {
// Check context cancellation
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
if err := shared.CheckContextCancellation(ctx, "file validation"); err != nil {
return nil, fmt.Errorf("context check during file validation: %w", err)
}
fileInfo, err := os.Stat(filePath)
if err != nil {
structErr := gibidiutils.WrapError(
err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSAccess,
structErr := shared.WrapError(
err,
shared.ErrorTypeFileSystem,
shared.CodeFSAccess,
"failed to stat file",
).WithFilePath(filePath)
gibidiutils.LogErrorf(structErr, "Failed to stat file %s", filePath)
shared.LogErrorf(structErr, "Failed to stat file %s", filePath)
return nil, structErr
}
// Check traditional size limit
if fileInfo.Size() > p.sizeLimit {
filesizeContext := map[string]interface{}{
c := map[string]any{
"file_size": fileInfo.Size(),
"size_limit": p.sizeLimit,
}
gibidiutils.LogErrorf(
gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationSize,
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", fileInfo.Size(), p.sizeLimit),
filePath,
filesizeContext,
),
"Skipping large file %s", filePath,
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeValidationSize,
fmt.Sprintf(shared.FileProcessingMsgSizeExceeds, fileInfo.Size(), p.sizeLimit),
filePath,
c,
)
return nil, fmt.Errorf("file too large")
shared.LogErrorf(structErr, "Skipping large file %s", filePath)
return nil, structErr
}
// Check resource limits
if err := p.resourceMonitor.ValidateFileProcessing(filePath, fileInfo.Size()); err != nil {
gibidiutils.LogErrorf(err, "Resource limit validation failed for file: %s", filePath)
shared.LogErrorf(err, "Resource limit validation failed for file: %s", filePath)
return nil, err
}
@@ -260,6 +220,7 @@ func (p *FileProcessor) getRelativePath(filePath string) string {
if err != nil {
return filePath // Fallback
}
return relPath
}
@@ -268,38 +229,74 @@ func (p *FileProcessor) processInMemoryWithContext(
ctx context.Context,
filePath, relPath string,
outCh chan<- WriteRequest,
) {
) error {
// Check context before reading
if p.checkContextCancellation(ctx, filePath, "") {
return
select {
case <-ctx.Done():
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"file processing canceled",
filePath,
nil,
)
shared.LogErrorf(structErr, "File processing canceled: %s", filePath)
return structErr
default:
}
// #nosec G304 - filePath is validated by walker
content, err := os.ReadFile(filePath)
content, err := os.ReadFile(filePath) // #nosec G304 - filePath is validated by walker
if err != nil {
structErr := gibidiutils.WrapError(
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingFileRead,
structErr := shared.WrapError(
err,
shared.ErrorTypeProcessing,
shared.CodeProcessingFileRead,
"failed to read file",
).WithFilePath(filePath)
gibidiutils.LogErrorf(structErr, "Failed to read file %s", filePath)
return
shared.LogErrorf(structErr, "Failed to read file %s", filePath)
return structErr
}
// Check context again after reading
if p.checkContextCancellation(ctx, filePath, "after read") {
return
select {
case <-ctx.Done():
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"file processing canceled after read",
filePath,
nil,
)
shared.LogErrorf(structErr, "File processing canceled after read: %s", filePath)
return structErr
default:
}
// Check context before sending output
if p.checkContextCancellation(ctx, filePath, "before output") {
return
}
// Try to send the result, but respect context cancellation
select {
case <-ctx.Done():
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"file processing canceled before output",
filePath,
nil,
)
shared.LogErrorf(structErr, "File processing canceled before output: %s", filePath)
outCh <- WriteRequest{
return structErr
case outCh <- WriteRequest{
Path: relPath,
Content: p.formatContent(relPath, string(content)),
IsStream: false,
Size: int64(len(content)),
}:
}
return nil
}
// processStreamingWithContext creates a streaming reader for large files with context awareness.
@@ -307,58 +304,87 @@ func (p *FileProcessor) processStreamingWithContext(
ctx context.Context,
filePath, relPath string,
outCh chan<- WriteRequest,
) {
size int64,
) error {
// Check context before creating reader
if p.checkContextCancellation(ctx, filePath, "before streaming") {
return
select {
case <-ctx.Done():
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"streaming processing canceled",
filePath,
nil,
)
shared.LogErrorf(structErr, "Streaming processing canceled: %s", filePath)
return structErr
default:
}
reader := p.createStreamReaderWithContext(ctx, filePath, relPath)
if reader == nil {
return // Error already logged
// Error already logged, create and return error
return shared.NewStructuredError(
shared.ErrorTypeProcessing,
shared.CodeProcessingFileRead,
"failed to create stream reader",
filePath,
nil,
)
}
// Check context before sending output
if p.checkContextCancellation(ctx, filePath, "before streaming output") {
// Close the reader to prevent file descriptor leak
if closer, ok := reader.(io.Closer); ok {
_ = closer.Close()
}
return
}
// Try to send the result, but respect context cancellation
select {
case <-ctx.Done():
structErr := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"streaming processing canceled before output",
filePath,
nil,
)
shared.LogErrorf(structErr, "Streaming processing canceled before output: %s", filePath)
outCh <- WriteRequest{
return structErr
case outCh <- WriteRequest{
Path: relPath,
Content: "", // Empty since content is in Reader
IsStream: true,
Reader: reader,
Size: size,
}:
}
return nil
}
// createStreamReaderWithContext creates a reader that combines header and file content with context awareness.
func (p *FileProcessor) createStreamReaderWithContext(ctx context.Context, filePath, relPath string) io.Reader {
func (p *FileProcessor) createStreamReaderWithContext(
ctx context.Context, filePath, relPath string,
) io.Reader {
// Check context before opening file
if p.checkContextCancellation(ctx, filePath, "before opening file") {
select {
case <-ctx.Done():
return nil
default:
}
// #nosec G304 - filePath is validated by walker
file, err := os.Open(filePath)
file, err := os.Open(filePath) // #nosec G304 - filePath is validated by walker
if err != nil {
structErr := gibidiutils.WrapError(
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingFileRead,
structErr := shared.WrapError(
err,
shared.ErrorTypeProcessing,
shared.CodeProcessingFileRead,
"failed to open file for streaming",
).WithFilePath(filePath)
gibidiutils.LogErrorf(structErr, "Failed to open file for streaming %s", filePath)
shared.LogErrorf(structErr, "Failed to open file for streaming %s", filePath)
return nil
}
header := p.formatHeader(relPath)
// Wrap in multiReaderCloser to ensure file is closed even on cancellation
return &multiReaderCloser{
reader: io.MultiReader(header, file),
closers: []io.Closer{file},
}
return newHeaderFileReader(header, file)
}
// formatContent formats the file content with header.
@@ -370,3 +396,66 @@ func (p *FileProcessor) formatContent(relPath, content string) string {
func (p *FileProcessor) formatHeader(relPath string) io.Reader {
return strings.NewReader(fmt.Sprintf("\n---\n%s\n", relPath))
}
// headerFileReader wraps a MultiReader and closes the file when EOF is reached.
type headerFileReader struct {
reader io.Reader
file *os.File
mu sync.Mutex
closed bool
}
// newHeaderFileReader creates a new headerFileReader.
func newHeaderFileReader(header io.Reader, file *os.File) *headerFileReader {
return &headerFileReader{
reader: io.MultiReader(header, file),
file: file,
}
}
// Read implements io.Reader and closes the file on EOF.
func (r *headerFileReader) Read(p []byte) (n int, err error) {
n, err = r.reader.Read(p)
if err == io.EOF {
r.closeFile()
// EOF is a sentinel value that must be passed through unchanged for io.Reader interface
return n, err //nolint:wrapcheck // EOF must not be wrapped
}
if err != nil {
return n, shared.WrapError(
err, shared.ErrorTypeIO, shared.CodeIORead,
"failed to read from header file reader",
)
}
return n, nil
}
// closeFile closes the file once.
func (r *headerFileReader) closeFile() {
r.mu.Lock()
defer r.mu.Unlock()
if !r.closed && r.file != nil {
if err := r.file.Close(); err != nil {
shared.LogError("Failed to close file", err)
}
r.closed = true
}
}
// Close implements io.Closer and ensures the underlying file is closed.
// This allows explicit cleanup when consumers stop reading before EOF.
func (r *headerFileReader) Close() error {
r.mu.Lock()
defer r.mu.Unlock()
if r.closed || r.file == nil {
return nil
}
err := r.file.Close()
if err != nil {
shared.LogError("Failed to close file", err)
}
r.closed = true
return err
}

763
fileproc/processor_test.go
View File

@@ -1,15 +1,84 @@
package fileproc_test
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/spf13/viper"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/fileproc"
"github.com/ivuorinen/gibidify/shared"
"github.com/ivuorinen/gibidify/testutil"
)
// writeTempConfig creates a temporary config file with the given YAML content
// and returns the directory path containing the config file.
func writeTempConfig(t *testing.T, content string) string {
t.Helper()
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {
t.Fatalf("Failed to create temp config: %v", err)
}
return dir
}
// collectWriteRequests runs a processing function and collects all WriteRequests.
// This helper wraps the common pattern of channel + goroutine + WaitGroup.
func collectWriteRequests(t *testing.T, process func(ch chan fileproc.WriteRequest)) []fileproc.WriteRequest {
t.Helper()
ch := make(chan fileproc.WriteRequest, 10)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
process(ch)
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
return results
}
// collectWriteRequestsWithContext runs a processing function with context and collects all WriteRequests.
func collectWriteRequestsWithContext(
ctx context.Context,
t *testing.T,
process func(ctx context.Context, ch chan fileproc.WriteRequest) error,
) ([]fileproc.WriteRequest, error) {
t.Helper()
ch := make(chan fileproc.WriteRequest, 10)
var processErr error
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
processErr = process(ctx, ch)
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
return results, processErr
}
func TestProcessFile(t *testing.T) {
// Reset and load default config to ensure proper file size limits
testutil.ResetViperConfig(t, "")
@@ -32,23 +101,20 @@ func TestProcessFile(t *testing.T) {
errTmpFile := tmpFile.Close()
if errTmpFile != nil {
t.Fatal(errTmpFile)
return
}
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
wg.Go(func() {
defer close(ch)
fileproc.ProcessFile(tmpFile.Name(), ch, "")
}()
wg.Wait()
close(ch)
})
var result string
for req := range ch {
result = req.Content
}
wg.Wait()
if !strings.Contains(result, tmpFile.Name()) {
t.Errorf("Output does not contain file path: %s", tmpFile.Name())
@@ -57,3 +123,686 @@ func TestProcessFile(t *testing.T) {
t.Errorf("Output does not contain file content: %s", content)
}
}
// TestNewFileProcessorWithMonitor tests processor creation with resource monitor.
func TestNewFileProcessorWithMonitor(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Create a resource monitor
monitor := fileproc.NewResourceMonitor()
defer monitor.Close()
processor := fileproc.NewFileProcessorWithMonitor("test_source", monitor)
if processor == nil {
t.Error("Expected processor but got nil")
}
// Exercise the processor to verify monitor integration
tmpFile, err := os.CreateTemp(t.TempDir(), "monitor_test")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpFile.Name())
if _, err := tmpFile.WriteString("test content"); err != nil {
t.Fatal(err)
}
if err := tmpFile.Close(); err != nil {
t.Fatal(err)
}
ctx := context.Background()
writeCh := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(writeCh)
if err := processor.ProcessWithContext(ctx, tmpFile.Name(), writeCh); err != nil {
t.Errorf("ProcessWithContext failed: %v", err)
}
})
// Drain channel first to avoid deadlock if producer sends multiple requests
requestCount := 0
for range writeCh {
requestCount++
}
// Wait for goroutine to finish after channel is drained
wg.Wait()
if requestCount == 0 {
t.Error("Expected at least one write request from processor")
}
}
// TestProcessFileWithMonitor tests file processing with resource monitoring.
func TestProcessFileWithMonitor(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Create temporary file
tmpFile, err := os.CreateTemp(t.TempDir(), "testfile_monitor_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
defer func() {
if err := os.Remove(tmpFile.Name()); err != nil {
t.Logf("Failed to remove temp file: %v", err)
}
}()
content := "Test content with monitor"
if _, err := tmpFile.WriteString(content); err != nil {
t.Fatalf(shared.TestMsgFailedToWriteContent, err)
}
if err := tmpFile.Close(); err != nil {
t.Fatalf(shared.TestMsgFailedToCloseFile, err)
}
// Create resource monitor
monitor := fileproc.NewResourceMonitor()
defer monitor.Close()
ch := make(chan fileproc.WriteRequest, 1)
ctx := context.Background()
// Test ProcessFileWithMonitor
var wg sync.WaitGroup
var result string
// Start reader goroutine first to prevent deadlock
wg.Go(func() {
for req := range ch {
result = req.Content
}
})
// Process the file
err = fileproc.ProcessFileWithMonitor(ctx, tmpFile.Name(), ch, "", monitor)
close(ch)
if err != nil {
t.Fatalf("ProcessFileWithMonitor failed: %v", err)
}
// Wait for reader to finish
wg.Wait()
if !strings.Contains(result, content) {
t.Error("Expected content not found in processed result")
}
}
const testContent = "package main\nfunc main() {}\n"
// TestProcess tests the basic Process function.
func TestProcess(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Create temporary directory
tmpDir := t.TempDir()
// Create test file with .go extension
testFile := filepath.Join(tmpDir, "test.go")
content := testContent
if err := os.WriteFile(testFile, []byte(content), 0o600); err != nil {
t.Fatalf(shared.TestMsgFailedToCreateTestFile, err)
}
processor := fileproc.NewFileProcessor(tmpDir)
ch := make(chan fileproc.WriteRequest, 10)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
// Process the specific file, not the directory
processor.Process(testFile, ch)
})
// Collect results
results := make([]fileproc.WriteRequest, 0, 1) // Pre-allocate with expected capacity
for req := range ch {
results = append(results, req)
}
wg.Wait()
if len(results) == 0 {
t.Error("Expected at least one processed file")
return
}
// Find our test file in results
found := false
for _, req := range results {
if strings.Contains(req.Path, shared.TestFileGo) && strings.Contains(req.Content, content) {
found = true
break
}
}
if !found {
t.Error("Test file not found in processed results")
}
}
// createLargeTestFile creates a large test file for streaming tests.
func createLargeTestFile(t *testing.T) *os.File {
t.Helper()
tmpFile, err := os.CreateTemp(t.TempDir(), "large_file_*.go")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
lineContent := "// Repeated comment line to exceed streaming threshold\n"
repeatCount := (1048576 / len(lineContent)) + 1000
largeContent := strings.Repeat(lineContent, repeatCount)
if _, err := tmpFile.WriteString(largeContent); err != nil {
t.Fatalf(shared.TestMsgFailedToWriteContent, err)
}
if err := tmpFile.Close(); err != nil {
t.Fatalf(shared.TestMsgFailedToCloseFile, err)
}
t.Logf("Created test file size: %d bytes", len(largeContent))
return tmpFile
}
// processFileForStreaming processes a file and returns streaming/inline requests.
func processFileForStreaming(t *testing.T, filePath string) (streamingReq, inlineReq *fileproc.WriteRequest) {
t.Helper()
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
fileproc.ProcessFile(filePath, ch, "")
})
var streamingRequest *fileproc.WriteRequest
var inlineRequest *fileproc.WriteRequest
for req := range ch {
if req.IsStream {
reqCopy := req
streamingRequest = &reqCopy
} else {
reqCopy := req
inlineRequest = &reqCopy
}
}
wg.Wait()
return streamingRequest, inlineRequest
}
// validateStreamingRequest validates a streaming request.
func validateStreamingRequest(t *testing.T, streamingRequest *fileproc.WriteRequest, tmpFile *os.File) {
t.Helper()
if streamingRequest.Reader == nil {
t.Error("Expected reader in streaming request")
}
if streamingRequest.Content != "" {
t.Error("Expected empty content for streaming request")
}
buffer := make([]byte, 1024)
n, err := streamingRequest.Reader.Read(buffer)
if err != nil && err != io.EOF {
t.Errorf("Failed to read from streaming request: %v", err)
}
content := string(buffer[:n])
if !strings.Contains(content, tmpFile.Name()) {
t.Error("Expected file path in streamed header content")
}
t.Log("Successfully triggered streaming for large file and tested reader")
}
// TestProcessorStreamingIntegration tests streaming functionality in processor.
func TestProcessorStreamingIntegration(t *testing.T) {
configDir := writeTempConfig(t, `
max_file_size_mb: 0.001
streaming_threshold_mb: 0.0001
`)
testutil.ResetViperConfig(t, configDir)
tmpFile := createLargeTestFile(t)
defer func() {
if err := os.Remove(tmpFile.Name()); err != nil {
t.Logf("Failed to remove temp file: %v", err)
}
}()
streamingRequest, inlineRequest := processFileForStreaming(t, tmpFile.Name())
if streamingRequest == nil && inlineRequest == nil {
t.Error("Expected either streaming or inline request but got none")
}
if streamingRequest != nil {
validateStreamingRequest(t, streamingRequest, tmpFile)
} else {
t.Log("File processed inline instead of streaming")
}
}
// TestProcessorContextCancellation tests context cancellation during processing.
func TestProcessorContextCancellation(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Create temporary directory with files
tmpDir := t.TempDir()
// Create multiple test files
for i := 0; i < 5; i++ {
testFile := filepath.Join(tmpDir, fmt.Sprintf("test%d.go", i))
content := testContent
if err := os.WriteFile(testFile, []byte(content), 0o600); err != nil {
t.Fatalf(shared.TestMsgFailedToCreateTestFile, err)
}
}
processor := fileproc.NewFileProcessor("test_source")
ch := make(chan fileproc.WriteRequest, 10)
// Use ProcessWithContext with immediate cancellation
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
// Error is expected due to cancellation
if err := processor.ProcessWithContext(ctx, tmpDir, ch); err != nil {
// Log error for debugging, but don't fail test since cancellation is expected
t.Logf("Expected error due to cancellation: %v", err)
}
})
// Collect results - should be minimal due to cancellation
results := make([]fileproc.WriteRequest, 0, 1) // Pre-allocate with expected capacity
for req := range ch {
results = append(results, req)
}
wg.Wait()
// With immediate cancellation, we might get 0 results
// This tests that cancellation is respected
t.Logf("Processed %d files with immediate cancellation", len(results))
}
// TestProcessorValidationEdgeCases tests edge cases in file validation.
func TestProcessorValidationEdgeCases(t *testing.T) {
configDir := writeTempConfig(t, `
max_file_size_mb: 0.001 # 1KB limit for testing
`)
testutil.ResetViperConfig(t, configDir)
tmpDir := t.TempDir()
// Test case 1: Non-existent file
nonExistentFile := filepath.Join(tmpDir, "does-not-exist.go")
processor := fileproc.NewFileProcessor(tmpDir)
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
processor.Process(nonExistentFile, ch)
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
// Should get no results due to file not existing
if len(results) > 0 {
t.Error("Expected no results for non-existent file")
}
// Test case 2: File that exceeds size limit
largeFile := filepath.Join(tmpDir, "large.go")
largeContent := strings.Repeat("// Large file content\n", 100) // > 1KB
if err := os.WriteFile(largeFile, []byte(largeContent), 0o600); err != nil {
t.Fatalf("Failed to create large file: %v", err)
}
ch2 := make(chan fileproc.WriteRequest, 1)
wg.Go(func() {
defer close(ch2)
processor.Process(largeFile, ch2)
})
results2 := make([]fileproc.WriteRequest, 0)
for req := range ch2 {
results2 = append(results2, req)
}
wg.Wait()
// Should get results because even large files are processed (just different strategy)
t.Logf("Large file processing results: %d", len(results2))
}
// TestProcessorContextCancellationDuringValidation tests context cancellation during file validation.
func TestProcessorContextCancellationDuringValidation(t *testing.T) {
testutil.ResetViperConfig(t, "")
tmpDir := t.TempDir()
testFile := filepath.Join(tmpDir, "test.go")
content := testContent
if err := os.WriteFile(testFile, []byte(content), 0o600); err != nil {
t.Fatalf(shared.TestMsgFailedToCreateTestFile, err)
}
processor := fileproc.NewFileProcessor(tmpDir)
// Create context that we'll cancel during processing
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
defer cancel()
// Let context expire
time.Sleep(1 * time.Millisecond)
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
if err := processor.ProcessWithContext(ctx, testFile, ch); err != nil {
t.Logf("ProcessWithContext error (may be expected): %v", err)
}
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
// Should get no results due to context cancellation
t.Logf("Results with canceled context: %d", len(results))
}
// TestProcessorInMemoryProcessingEdgeCases tests edge cases in in-memory processing.
func TestProcessorInMemoryProcessingEdgeCases(t *testing.T) {
testutil.ResetViperConfig(t, "")
tmpDir := t.TempDir()
// Test with empty file
emptyFile := filepath.Join(tmpDir, "empty.go")
if err := os.WriteFile(emptyFile, []byte(""), 0o600); err != nil {
t.Fatalf("Failed to create empty file: %v", err)
}
processor := fileproc.NewFileProcessor(tmpDir)
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
processor.Process(emptyFile, ch)
})
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
}
wg.Wait()
if len(results) != 1 {
t.Errorf("Expected 1 result for empty file, got %d", len(results))
}
if len(results) > 0 {
result := results[0]
if result.Path == "" {
t.Error("Expected path in result for empty file")
}
// Empty file should still be processed
}
}
// TestProcessorStreamingEdgeCases tests edge cases in streaming processing.
func TestProcessorStreamingEdgeCases(t *testing.T) {
testutil.ResetViperConfig(t, "")
tmpDir := t.TempDir()
// Create a file larger than streaming threshold but test error conditions
largeFile := filepath.Join(tmpDir, "large_stream.go")
largeContent := strings.Repeat("// Large streaming file content line\n", 50000) // > 1MB
if err := os.WriteFile(largeFile, []byte(largeContent), 0o600); err != nil {
t.Fatalf("Failed to create large file: %v", err)
}
processor := fileproc.NewFileProcessor(tmpDir)
// Test with context that gets canceled during streaming
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
// Start processing
// Error is expected due to cancellation
if err := processor.ProcessWithContext(ctx, largeFile, ch); err != nil {
// Log error for debugging, but don't fail test since cancellation is expected
t.Logf("Expected error due to cancellation: %v", err)
}
})
// Cancel context after a very short time
go func() {
time.Sleep(1 * time.Millisecond)
cancel()
}()
results := make([]fileproc.WriteRequest, 0)
for req := range ch {
results = append(results, req)
// If we get a streaming request, try to read from it with canceled context
if req.IsStream && req.Reader != nil {
buffer := make([]byte, 1024)
_, err := req.Reader.Read(buffer)
if err != nil && err != io.EOF {
t.Logf("Expected error reading from canceled stream: %v", err)
}
}
}
wg.Wait()
t.Logf("Results with streaming context cancellation: %d", len(results))
}
// Benchmarks for processor hot paths
// BenchmarkProcessFileInline benchmarks inline file processing for small files.
func BenchmarkProcessFileInline(b *testing.B) {
// Initialize config for file processing
viper.Reset()
config.LoadConfig()
// Create a small test file
tmpFile, err := os.CreateTemp(b.TempDir(), "bench_inline_*.go")
if err != nil {
b.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
content := strings.Repeat("// Inline benchmark content\n", 100) // ~2.6KB
if _, err := tmpFile.WriteString(content); err != nil {
b.Fatalf(shared.TestMsgFailedToWriteContent, err)
}
if err := tmpFile.Close(); err != nil {
b.Fatalf(shared.TestMsgFailedToCloseFile, err)
}
b.ResetTimer()
for b.Loop() {
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
fileproc.ProcessFile(tmpFile.Name(), ch, "")
})
for req := range ch {
_ = req // Drain channel
}
wg.Wait()
}
}
// BenchmarkProcessFileStreaming benchmarks streaming file processing for large files.
func BenchmarkProcessFileStreaming(b *testing.B) {
// Initialize config for file processing
viper.Reset()
config.LoadConfig()
// Create a large test file that triggers streaming
tmpFile, err := os.CreateTemp(b.TempDir(), "bench_streaming_*.go")
if err != nil {
b.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
// Create content larger than streaming threshold (1MB)
lineContent := "// Streaming benchmark content line that will be repeated\n"
repeatCount := (1048576 / len(lineContent)) + 1000
content := strings.Repeat(lineContent, repeatCount)
if _, err := tmpFile.WriteString(content); err != nil {
b.Fatalf(shared.TestMsgFailedToWriteContent, err)
}
if err := tmpFile.Close(); err != nil {
b.Fatalf(shared.TestMsgFailedToCloseFile, err)
}
b.ResetTimer()
for b.Loop() {
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
fileproc.ProcessFile(tmpFile.Name(), ch, "")
})
for req := range ch {
// If streaming, read some content to exercise the reader
if req.IsStream && req.Reader != nil {
buffer := make([]byte, 4096)
for {
_, err := req.Reader.Read(buffer)
if err != nil {
break
}
}
}
}
wg.Wait()
}
}
// BenchmarkProcessorWithContext benchmarks ProcessWithContext for a single file.
func BenchmarkProcessorWithContext(b *testing.B) {
tmpDir := b.TempDir()
testFile := filepath.Join(tmpDir, "bench_context.go")
content := strings.Repeat("// Benchmark file content\n", 50)
if err := os.WriteFile(testFile, []byte(content), 0o600); err != nil {
b.Fatalf(shared.TestMsgFailedToCreateTestFile, err)
}
processor := fileproc.NewFileProcessor(tmpDir)
ctx := context.Background()
b.ResetTimer()
for b.Loop() {
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
_ = processor.ProcessWithContext(ctx, testFile, ch)
})
for req := range ch {
_ = req // Drain channel
}
wg.Wait()
}
}
// BenchmarkProcessorWithMonitor benchmarks processing with resource monitoring.
func BenchmarkProcessorWithMonitor(b *testing.B) {
tmpDir := b.TempDir()
testFile := filepath.Join(tmpDir, "bench_monitor.go")
content := strings.Repeat("// Benchmark file content with monitor\n", 50)
if err := os.WriteFile(testFile, []byte(content), 0o600); err != nil {
b.Fatalf(shared.TestMsgFailedToCreateTestFile, err)
}
monitor := fileproc.NewResourceMonitor()
defer monitor.Close()
processor := fileproc.NewFileProcessorWithMonitor(tmpDir, monitor)
ctx := context.Background()
b.ResetTimer()
for b.Loop() {
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
_ = processor.ProcessWithContext(ctx, testFile, ch)
})
for req := range ch {
_ = req // Drain channel
}
wg.Wait()
}
}
// BenchmarkProcessorConcurrent benchmarks concurrent file processing.
func BenchmarkProcessorConcurrent(b *testing.B) {
tmpDir := b.TempDir()
// Create multiple test files
testFiles := make([]string, 10)
for i := 0; i < 10; i++ {
testFiles[i] = filepath.Join(tmpDir, fmt.Sprintf("bench_concurrent_%d.go", i))
content := strings.Repeat(fmt.Sprintf("// Concurrent file %d content\n", i), 50)
if err := os.WriteFile(testFiles[i], []byte(content), 0o600); err != nil {
b.Fatalf(shared.TestMsgFailedToCreateTestFile, err)
}
}
processor := fileproc.NewFileProcessor(tmpDir)
ctx := context.Background()
fileCount := len(testFiles)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
i := 0
for pb.Next() {
testFile := testFiles[i%fileCount]
ch := make(chan fileproc.WriteRequest, 1)
var wg sync.WaitGroup
wg.Go(func() {
defer close(ch)
_ = processor.ProcessWithContext(ctx, testFile, ch)
})
for req := range ch {
_ = req // Drain channel
}
wg.Wait()
i++
}
})
}

25
fileproc/registry.go
View File

@@ -5,6 +5,8 @@ import (
"path/filepath"
"strings"
"sync"
"github.com/ivuorinen/gibidify/shared"
)
const minExtensionLength = 2
@@ -52,9 +54,9 @@ func initRegistry() *FileTypeRegistry {
imageExts: getImageExtensions(),
binaryExts: getBinaryExtensions(),
languageMap: getLanguageMap(),
extCache: make(map[string]string, 1000), // Cache for extension normalization
resultCache: make(map[string]FileTypeResult, 500), // Cache for type results
maxCacheSize: 500,
extCache: make(map[string]string, shared.FileTypeRegistryMaxCacheSize),
resultCache: make(map[string]FileTypeResult, shared.FileTypeRegistryMaxCacheSize),
maxCacheSize: shared.FileTypeRegistryMaxCacheSize,
}
}
@@ -63,25 +65,28 @@ func getRegistry() *FileTypeRegistry {
registryOnce.Do(func() {
registry = initRegistry()
})
return registry
}
// GetDefaultRegistry returns the default file type registry.
func GetDefaultRegistry() *FileTypeRegistry {
// DefaultRegistry returns the default file type registry.
func DefaultRegistry() *FileTypeRegistry {
return getRegistry()
}
// GetStats returns a copy of the current registry statistics.
func (r *FileTypeRegistry) GetStats() RegistryStats {
// Stats returns a copy of the current registry statistics.
func (r *FileTypeRegistry) Stats() RegistryStats {
r.cacheMutex.RLock()
defer r.cacheMutex.RUnlock()
return r.stats
}
// GetCacheInfo returns current cache size information.
func (r *FileTypeRegistry) GetCacheInfo() (extCacheSize, resultCacheSize, maxCacheSize int) {
// CacheInfo returns current cache size information.
func (r *FileTypeRegistry) CacheInfo() (extCacheSize, resultCacheSize, maxCacheSize int) {
r.cacheMutex.RLock()
defer r.cacheMutex.RUnlock()
return len(r.extCache), len(r.resultCache), r.maxCacheSize
}
@@ -101,7 +106,9 @@ func normalizeExtension(filename string) string {
func isSpecialFile(filename string, extensions map[string]bool) bool {
if filepath.Ext(filename) == "" {
basename := strings.ToLower(filepath.Base(filename))
return extensions[basename]
}
return false
}

15
fileproc/resource_monitor_concurrency.go
View File

@@ -1,7 +1,9 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
"context"
"fmt"
"sync/atomic"
"time"
)
@@ -26,7 +28,7 @@ func (rm *ResourceMonitor) AcquireReadSlot(ctx context.Context) error {
// Wait and retry
select {
case <-ctx.Done():
return ctx.Err()
return fmt.Errorf("context canceled while waiting for read slot: %w", ctx.Err())
case <-time.After(time.Millisecond):
// Continue loop
}
@@ -45,17 +47,22 @@ func (rm *ResourceMonitor) ReleaseReadSlot() {
// CreateFileProcessingContext creates a context with file processing timeout.
func (rm *ResourceMonitor) CreateFileProcessingContext(parent context.Context) (context.Context, context.CancelFunc) {
if !rm.enabled || rm.fileProcessingTimeout <= 0 {
// No-op cancel function - monitoring disabled or no timeout configured
return parent, func() {}
}
return context.WithTimeout(parent, rm.fileProcessingTimeout)
}
// CreateOverallProcessingContext creates a context with overall processing timeout.
func (rm *ResourceMonitor) CreateOverallProcessingContext(
parent context.Context,
) (context.Context, context.CancelFunc) {
func (rm *ResourceMonitor) CreateOverallProcessingContext(parent context.Context) (
context.Context,
context.CancelFunc,
) {
if !rm.enabled || rm.overallTimeout <= 0 {
// No-op cancel function - monitoring disabled or no timeout configured
return parent, func() {}
}
return context.WithTimeout(parent, rm.overallTimeout)
}

4
fileproc/resource_monitor_concurrency_test.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/ivuorinen/gibidify/testutil"
)
func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) {
func TestResourceMonitorConcurrentReadsLimit(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set a low concurrent reads limit for testing
@@ -58,7 +58,7 @@ func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) {
rm.ReleaseReadSlot()
}
func TestResourceMonitor_TimeoutContexts(t *testing.T) {
func TestResourceMonitorTimeoutContexts(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set short timeouts for testing

6
fileproc/resource_monitor_integration_test.go
View File

@@ -11,7 +11,7 @@ import (
"github.com/ivuorinen/gibidify/testutil"
)
func TestResourceMonitor_Integration(t *testing.T) {
func TestResourceMonitorIntegration(t *testing.T) {
// Create temporary test directory
tempDir := t.TempDir()
@@ -47,6 +47,7 @@ func TestResourceMonitor_Integration(t *testing.T) {
err = rm.ValidateFileProcessing(filePath, fileInfo.Size())
if err != nil {
t.Errorf("Failed to validate file %s: %v", filePath, err)
continue
}
@@ -54,6 +55,7 @@ func TestResourceMonitor_Integration(t *testing.T) {
err = rm.AcquireReadSlot(ctx)
if err != nil {
t.Errorf("Failed to acquire read slot for %s: %v", filePath, err)
continue
}
@@ -71,7 +73,7 @@ func TestResourceMonitor_Integration(t *testing.T) {
}
// Verify final metrics
metrics := rm.GetMetrics()
metrics := rm.Metrics()
if metrics.FilesProcessed != int64(len(testFiles)) {
t.Errorf("Expected %d files processed, got %d", len(testFiles), metrics.FilesProcessed)
}

29
fileproc/resource_monitor_metrics.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
@@ -5,9 +6,7 @@ import (
"sync/atomic"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// RecordFileProcessed records that a file has been successfully processed.
@@ -18,8 +17,8 @@ func (rm *ResourceMonitor) RecordFileProcessed(fileSize int64) {
}
}
// GetMetrics returns current resource usage metrics.
func (rm *ResourceMonitor) GetMetrics() ResourceMetrics {
// Metrics returns current resource usage metrics.
func (rm *ResourceMonitor) Metrics() ResourceMetrics {
if !rm.enableResourceMon {
return ResourceMetrics{}
}
@@ -54,10 +53,11 @@ func (rm *ResourceMonitor) GetMetrics() ResourceMetrics {
FilesProcessed: filesProcessed,
TotalSizeProcessed: totalSize,
ConcurrentReads: atomic.LoadInt64(&rm.concurrentReads),
MaxConcurrentReads: int64(rm.maxConcurrentReads),
ProcessingDuration: duration,
AverageFileSize: avgFileSize,
ProcessingRate: processingRate,
MemoryUsageMB: gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0) / 1024 / 1024,
MemoryUsageMB: shared.BytesToMB(m.Alloc),
MaxMemoryUsageMB: int64(rm.hardMemoryLimitMB),
ViolationsDetected: violations,
DegradationActive: rm.degradationActive,
@@ -68,19 +68,16 @@ func (rm *ResourceMonitor) GetMetrics() ResourceMetrics {
// LogResourceInfo logs current resource limit configuration.
func (rm *ResourceMonitor) LogResourceInfo() {
logger := shared.GetLogger()
if rm.enabled {
logrus.Infof(
"Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds",
rm.maxFiles,
rm.maxTotalSize/1024/1024,
int(rm.fileProcessingTimeout.Seconds()),
int(rm.overallTimeout.Seconds()),
)
logrus.Infof("Resource limits: maxConcurrentReads=%d, rateLimitFPS=%d, hardMemoryMB=%d",
logger.Infof("Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds",
rm.maxFiles, rm.maxTotalSize/int64(shared.BytesPerMB), int(rm.fileProcessingTimeout.Seconds()),
int(rm.overallTimeout.Seconds()))
logger.Infof("Resource limits: maxConcurrentReads=%d, rateLimitFPS=%d, hardMemoryMB=%d",
rm.maxConcurrentReads, rm.rateLimitFilesPerSec, rm.hardMemoryLimitMB)
logrus.Infof("Resource features: gracefulDegradation=%v, monitoring=%v",
logger.Infof("Resource features: gracefulDegradation=%v, monitoring=%v",
rm.enableGracefulDegr, rm.enableResourceMon)
} else {
logrus.Info("Resource limits disabled")
logger.Info("Resource limits disabled")
}
}

4
fileproc/resource_monitor_metrics_test.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/ivuorinen/gibidify/testutil"
)
func TestResourceMonitor_Metrics(t *testing.T) {
func TestResourceMonitorMetrics(t *testing.T) {
testutil.ResetViperConfig(t, "")
viper.Set("resourceLimits.enabled", true)
@@ -23,7 +23,7 @@ func TestResourceMonitor_Metrics(t *testing.T) {
rm.RecordFileProcessed(2000)
rm.RecordFileProcessed(500)
metrics := rm.GetMetrics()
metrics := rm.Metrics()
// Verify metrics
if metrics.FilesProcessed != 3 {

23
fileproc/resource_monitor_rate_limiting.go
View File

@@ -1,10 +1,12 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
"context"
"fmt"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/shared"
)
// WaitForRateLimit waits for rate limiting if enabled.
@@ -15,22 +17,29 @@ func (rm *ResourceMonitor) WaitForRateLimit(ctx context.Context) error {
select {
case <-ctx.Done():
return ctx.Err()
return fmt.Errorf("context canceled while waiting for rate limit: %w", ctx.Err())
case <-rm.rateLimitChan:
return nil
case <-time.After(time.Second): // Fallback timeout
logrus.Warn("Rate limiting timeout exceeded, continuing without rate limit")
logger := shared.GetLogger()
logger.Warn("Rate limiting timeout exceeded, continuing without rate limit")
return nil
}
}
// rateLimiterRefill refills the rate limiting channel periodically.
func (rm *ResourceMonitor) rateLimiterRefill() {
for range rm.rateLimiter.C {
for {
select {
case rm.rateLimitChan <- struct{}{}:
default:
// Channel is full, skip
case <-rm.done:
return
case <-rm.rateLimiter.C:
select {
case rm.rateLimitChan <- struct{}{}:
default:
// Channel is full, skip
}
}
}
}

2
fileproc/resource_monitor_rate_limiting_test.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/ivuorinen/gibidify/testutil"
)
func TestResourceMonitor_RateLimiting(t *testing.T) {
func TestResourceMonitorRateLimiting(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Enable rate limiting with a low rate for testing

18
fileproc/resource_monitor_state.go
View File

@@ -1,9 +1,11 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
// IsEmergencyStopActive returns whether emergency stop is active.
func (rm *ResourceMonitor) IsEmergencyStopActive() bool {
rm.mu.RLock()
defer rm.mu.RUnlock()
return rm.emergencyStopRequested
}
@@ -11,11 +13,27 @@ func (rm *ResourceMonitor) IsEmergencyStopActive() bool {
func (rm *ResourceMonitor) IsDegradationActive() bool {
rm.mu.RLock()
defer rm.mu.RUnlock()
return rm.degradationActive
}
// Close cleans up the resource monitor.
func (rm *ResourceMonitor) Close() {
rm.mu.Lock()
defer rm.mu.Unlock()
// Prevent multiple closes
if rm.closed {
return
}
rm.closed = true
// Signal goroutines to stop
if rm.done != nil {
close(rm.done)
}
// Stop the ticker
if rm.rateLimiter != nil {
rm.rateLimiter.Stop()
}

40
fileproc/resource_monitor_types.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
@@ -5,6 +6,7 @@ import (
"time"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/shared"
)
// ResourceMonitor monitors resource usage and enforces limits to prevent DoS attacks.
@@ -31,12 +33,14 @@ type ResourceMonitor struct {
// Rate limiting
rateLimiter *time.Ticker
rateLimitChan chan struct{}
done chan struct{} // Signal to stop goroutines
// Synchronization
mu sync.RWMutex
violationLogged map[string]bool
degradationActive bool
emergencyStopRequested bool
closed bool
}
// ResourceMetrics holds comprehensive resource usage metrics.
@@ -44,6 +48,7 @@ type ResourceMetrics struct {
FilesProcessed int64 `json:"files_processed"`
TotalSizeProcessed int64 `json:"total_size_processed"`
ConcurrentReads int64 `json:"concurrent_reads"`
MaxConcurrentReads int64 `json:"max_concurrent_reads"`
ProcessingDuration time.Duration `json:"processing_duration"`
AverageFileSize float64 `json:"average_file_size"`
ProcessingRate float64 `json:"processing_rate_files_per_sec"`
@@ -57,31 +62,32 @@ type ResourceMetrics struct {
// ResourceViolation represents a detected resource limit violation.
type ResourceViolation struct {
Type string `json:"type"`
Message string `json:"message"`
Current interface{} `json:"current"`
Limit interface{} `json:"limit"`
Timestamp time.Time `json:"timestamp"`
Context map[string]interface{} `json:"context"`
Type string `json:"type"`
Message string `json:"message"`
Current any `json:"current"`
Limit any `json:"limit"`
Timestamp time.Time `json:"timestamp"`
Context map[string]any `json:"context"`
}
// NewResourceMonitor creates a new resource monitor with configuration.
func NewResourceMonitor() *ResourceMonitor {
rm := &ResourceMonitor{
enabled: config.GetResourceLimitsEnabled(),
maxFiles: config.GetMaxFiles(),
maxTotalSize: config.GetMaxTotalSize(),
fileProcessingTimeout: time.Duration(config.GetFileProcessingTimeoutSec()) * time.Second,
overallTimeout: time.Duration(config.GetOverallTimeoutSec()) * time.Second,
maxConcurrentReads: config.GetMaxConcurrentReads(),
rateLimitFilesPerSec: config.GetRateLimitFilesPerSec(),
hardMemoryLimitMB: config.GetHardMemoryLimitMB(),
enableGracefulDegr: config.GetEnableGracefulDegradation(),
enableResourceMon: config.GetEnableResourceMonitoring(),
enabled: config.ResourceLimitsEnabled(),
maxFiles: config.MaxFiles(),
maxTotalSize: config.MaxTotalSize(),
fileProcessingTimeout: time.Duration(config.FileProcessingTimeoutSec()) * time.Second,
overallTimeout: time.Duration(config.OverallTimeoutSec()) * time.Second,
maxConcurrentReads: config.MaxConcurrentReads(),
rateLimitFilesPerSec: config.RateLimitFilesPerSec(),
hardMemoryLimitMB: config.HardMemoryLimitMB(),
enableGracefulDegr: config.EnableGracefulDegradation(),
enableResourceMon: config.EnableResourceMonitoring(),
startTime: time.Now(),
lastRateLimitCheck: time.Now(),
violationLogged: make(map[string]bool),
hardMemoryLimitBytes: int64(config.GetHardMemoryLimitMB()) * 1024 * 1024,
hardMemoryLimitBytes: int64(config.HardMemoryLimitMB()) * int64(shared.BytesPerMB),
done: make(chan struct{}),
}
// Initialize rate limiter if rate limiting is enabled

92
fileproc/resource_monitor_types_test.go
View File

@@ -7,11 +7,11 @@ import (
"github.com/spf13/viper"
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/shared"
"github.com/ivuorinen/gibidify/testutil"
)
func TestResourceMonitor_NewResourceMonitor(t *testing.T) {
func TestResourceMonitorNewResourceMonitor(t *testing.T) {
// Reset viper for clean test state
testutil.ResetViperConfig(t, "")
@@ -25,24 +25,24 @@ func TestResourceMonitor_NewResourceMonitor(t *testing.T) {
t.Error("Expected resource monitor to be enabled by default")
}
if rm.maxFiles != config.DefaultMaxFiles {
t.Errorf("Expected maxFiles to be %d, got %d", config.DefaultMaxFiles, rm.maxFiles)
if rm.maxFiles != shared.ConfigMaxFilesDefault {
t.Errorf("Expected maxFiles to be %d, got %d", shared.ConfigMaxFilesDefault, rm.maxFiles)
}
if rm.maxTotalSize != config.DefaultMaxTotalSize {
t.Errorf("Expected maxTotalSize to be %d, got %d", config.DefaultMaxTotalSize, rm.maxTotalSize)
if rm.maxTotalSize != shared.ConfigMaxTotalSizeDefault {
t.Errorf("Expected maxTotalSize to be %d, got %d", shared.ConfigMaxTotalSizeDefault, rm.maxTotalSize)
}
if rm.fileProcessingTimeout != time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second {
if rm.fileProcessingTimeout != time.Duration(shared.ConfigFileProcessingTimeoutSecDefault)*time.Second {
t.Errorf("Expected fileProcessingTimeout to be %v, got %v",
time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second, rm.fileProcessingTimeout)
time.Duration(shared.ConfigFileProcessingTimeoutSecDefault)*time.Second, rm.fileProcessingTimeout)
}
// Clean up
rm.Close()
}
func TestResourceMonitor_DisabledResourceLimits(t *testing.T) {
func TestResourceMonitorDisabledResourceLimits(t *testing.T) {
// Reset viper for clean test state
testutil.ResetViperConfig(t, "")
@@ -72,3 +72,77 @@ func TestResourceMonitor_DisabledResourceLimits(t *testing.T) {
t.Errorf("Expected no error when rate limiting disabled, got %v", err)
}
}
// TestResourceMonitorStateQueries tests state query functions.
func TestResourceMonitorStateQueries(t *testing.T) {
testutil.ResetViperConfig(t, "")
rm := NewResourceMonitor()
defer rm.Close()
// Test IsEmergencyStopActive - should be false initially
if rm.IsEmergencyStopActive() {
t.Error("Expected emergency stop to be inactive initially")
}
// Test IsDegradationActive - should be false initially
if rm.IsDegradationActive() {
t.Error("Expected degradation mode to be inactive initially")
}
}
// TestResourceMonitorIsEmergencyStopActive tests the IsEmergencyStopActive method.
func TestResourceMonitorIsEmergencyStopActive(t *testing.T) {
testutil.ResetViperConfig(t, "")
rm := NewResourceMonitor()
defer rm.Close()
// Test initial state
active := rm.IsEmergencyStopActive()
if active {
t.Error("Expected emergency stop to be inactive initially")
}
// The method should return a consistent value on multiple calls
for i := 0; i < 5; i++ {
if rm.IsEmergencyStopActive() != active {
t.Error("IsEmergencyStopActive should return consistent values")
}
}
}
// TestResourceMonitorIsDegradationActive tests the IsDegradationActive method.
func TestResourceMonitorIsDegradationActive(t *testing.T) {
testutil.ResetViperConfig(t, "")
rm := NewResourceMonitor()
defer rm.Close()
// Test initial state
active := rm.IsDegradationActive()
if active {
t.Error("Expected degradation mode to be inactive initially")
}
// The method should return a consistent value on multiple calls
for i := 0; i < 5; i++ {
if rm.IsDegradationActive() != active {
t.Error("IsDegradationActive should return consistent values")
}
}
}
// TestResourceMonitorClose tests the Close method.
func TestResourceMonitorClose(t *testing.T) {
testutil.ResetViperConfig(t, "")
rm := NewResourceMonitor()
// Close should not panic
rm.Close()
// Multiple closes should be safe
rm.Close()
rm.Close()
}

174
fileproc/resource_monitor_validation.go
View File

@@ -1,3 +1,4 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
@@ -5,9 +6,7 @@ import (
"sync/atomic"
"time"
"github.com/sirupsen/logrus"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// ValidateFileProcessing checks if a file can be processed based on resource limits.
@@ -21,12 +20,12 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
// Check if emergency stop is active
if rm.emergencyStopRequested {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitMemory,
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitMemory,
"processing stopped due to emergency memory condition",
filePath,
map[string]interface{}{
map[string]any{
"emergency_stop_active": true,
},
)
@@ -35,12 +34,12 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
// Check file count limit
currentFiles := atomic.LoadInt64(&rm.filesProcessed)
if int(currentFiles) >= rm.maxFiles {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitFiles,
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitFiles,
"maximum file count limit exceeded",
filePath,
map[string]interface{}{
map[string]any{
"current_files": currentFiles,
"max_files": rm.maxFiles,
},
@@ -50,12 +49,12 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
// Check total size limit
currentTotalSize := atomic.LoadInt64(&rm.totalSizeProcessed)
if currentTotalSize+fileSize > rm.maxTotalSize {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitTotalSize,
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTotalSize,
"maximum total size limit would be exceeded",
filePath,
map[string]interface{}{
map[string]any{
"current_total_size": currentTotalSize,
"file_size": fileSize,
"max_total_size": rm.maxTotalSize,
@@ -65,12 +64,12 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
// Check overall timeout
if time.Since(rm.startTime) > rm.overallTimeout {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitTimeout,
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitTimeout,
"overall processing timeout exceeded",
filePath,
map[string]interface{}{
map[string]any{
"processing_duration": time.Since(rm.startTime),
"overall_timeout": rm.overallTimeout,
},
@@ -88,60 +87,93 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error {
var m runtime.MemStats
runtime.ReadMemStats(&m)
currentMemory := gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0)
currentMemory := shared.SafeUint64ToInt64WithDefault(m.Alloc, 0)
if currentMemory > rm.hardMemoryLimitBytes {
rm.mu.Lock()
defer rm.mu.Unlock()
// Log violation if not already logged
violationKey := "hard_memory_limit"
if !rm.violationLogged[violationKey] {
logrus.Errorf("Hard memory limit exceeded: %dMB > %dMB",
currentMemory/1024/1024, rm.hardMemoryLimitMB)
rm.violationLogged[violationKey] = true
}
if rm.enableGracefulDegr {
// Force garbage collection
runtime.GC()
// Check again after GC
runtime.ReadMemStats(&m)
currentMemory = gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0)
if currentMemory > rm.hardMemoryLimitBytes {
// Still over limit, activate emergency stop
rm.emergencyStopRequested = true
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitMemory,
"hard memory limit exceeded, emergency stop activated",
"",
map[string]interface{}{
"current_memory_mb": currentMemory / 1024 / 1024,
"limit_mb": rm.hardMemoryLimitMB,
"emergency_stop": true,
},
)
}
// Memory freed by GC, continue with degradation
rm.degradationActive = true
logrus.Info("Memory freed by garbage collection, continuing with degradation mode")
} else {
// No graceful degradation, hard stop
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeResourceLimitMemory,
"hard memory limit exceeded",
"",
map[string]interface{}{
"current_memory_mb": currentMemory / 1024 / 1024,
"limit_mb": rm.hardMemoryLimitMB,
},
)
}
if currentMemory <= rm.hardMemoryLimitBytes {
return nil
}
return rm.handleMemoryLimitExceeded(currentMemory)
}
// handleMemoryLimitExceeded handles the case when hard memory limit is exceeded.
func (rm *ResourceMonitor) handleMemoryLimitExceeded(currentMemory int64) error {
rm.mu.Lock()
defer rm.mu.Unlock()
rm.logMemoryViolation(currentMemory)
if !rm.enableGracefulDegr {
return rm.createHardMemoryLimitError(currentMemory, false)
}
return rm.tryGracefulRecovery(currentMemory)
}
// logMemoryViolation logs memory limit violation if not already logged.
func (rm *ResourceMonitor) logMemoryViolation(currentMemory int64) {
violationKey := "hard_memory_limit"
// Ensure map is initialized
if rm.violationLogged == nil {
rm.violationLogged = make(map[string]bool)
}
if rm.violationLogged[violationKey] {
return
}
logger := shared.GetLogger()
logger.Errorf("Hard memory limit exceeded: %dMB > %dMB",
currentMemory/int64(shared.BytesPerMB), rm.hardMemoryLimitMB)
rm.violationLogged[violationKey] = true
}
// tryGracefulRecovery attempts graceful recovery by forcing GC.
func (rm *ResourceMonitor) tryGracefulRecovery(_ int64) error {
// Force garbage collection
runtime.GC()
// Check again after GC
var m runtime.MemStats
runtime.ReadMemStats(&m)
newMemory := shared.SafeUint64ToInt64WithDefault(m.Alloc, 0)
if newMemory > rm.hardMemoryLimitBytes {
// Still over limit, activate emergency stop
rm.emergencyStopRequested = true
return rm.createHardMemoryLimitError(newMemory, true)
}
// Memory freed by GC, continue with degradation
rm.degradationActive = true
logger := shared.GetLogger()
logger.Info("Memory freed by garbage collection, continuing with degradation mode")
return nil
}
// createHardMemoryLimitError creates a structured error for memory limit exceeded.
func (rm *ResourceMonitor) createHardMemoryLimitError(currentMemory int64, emergencyStop bool) error {
message := "hard memory limit exceeded"
if emergencyStop {
message = "hard memory limit exceeded, emergency stop activated"
}
context := map[string]any{
"current_memory_mb": currentMemory / int64(shared.BytesPerMB),
"limit_mb": rm.hardMemoryLimitMB,
}
if emergencyStop {
context["emergency_stop"] = true
}
return shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeResourceLimitMemory,
message,
"",
context,
)
}

149
fileproc/resource_monitor_validation_test.go
View File

@@ -2,19 +2,46 @@ package fileproc
import (
"errors"
"strings"
"testing"
"github.com/spf13/viper"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
"github.com/ivuorinen/gibidify/testutil"
)
func TestResourceMonitor_FileCountLimit(t *testing.T) {
// assertStructuredError verifies that an error is a StructuredError with the expected code.
func assertStructuredError(t *testing.T, err error, expectedCode string) {
t.Helper()
structErr := &shared.StructuredError{}
ok := errors.As(err, &structErr)
if !ok {
t.Errorf("Expected StructuredError, got %T", err)
} else if structErr.Code != expectedCode {
t.Errorf("Expected error code %s, got %s", expectedCode, structErr.Code)
}
}
// validateMemoryLimitError validates that an error is a proper memory limit StructuredError.
func validateMemoryLimitError(t *testing.T, err error) {
t.Helper()
structErr := &shared.StructuredError{}
if errors.As(err, &structErr) {
if structErr.Code != shared.CodeResourceLimitMemory {
t.Errorf("Expected memory limit error code, got %s", structErr.Code)
}
} else {
t.Errorf("Expected StructuredError, got %T", err)
}
}
func TestResourceMonitorFileCountLimit(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set a very low file count limit for testing
viper.Set("resourceLimits.enabled", true)
viper.Set(shared.TestCfgResourceLimitsEnabled, true)
viper.Set("resourceLimits.maxFiles", 2)
rm := NewResourceMonitor()
@@ -41,20 +68,14 @@ func TestResourceMonitor_FileCountLimit(t *testing.T) {
}
// Verify it's the correct error type
var structErr *gibidiutils.StructuredError
ok := errors.As(err, &structErr)
if !ok {
t.Errorf("Expected StructuredError, got %T", err)
} else if structErr.Code != gibidiutils.CodeResourceLimitFiles {
t.Errorf("Expected error code %s, got %s", gibidiutils.CodeResourceLimitFiles, structErr.Code)
}
assertStructuredError(t, err, shared.CodeResourceLimitFiles)
}
func TestResourceMonitor_TotalSizeLimit(t *testing.T) {
func TestResourceMonitorTotalSizeLimit(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set a low total size limit for testing (1KB)
viper.Set("resourceLimits.enabled", true)
viper.Set(shared.TestCfgResourceLimitsEnabled, true)
viper.Set("resourceLimits.maxTotalSize", 1024)
rm := NewResourceMonitor()
@@ -81,11 +102,103 @@ func TestResourceMonitor_TotalSizeLimit(t *testing.T) {
}
// Verify it's the correct error type
var structErr *gibidiutils.StructuredError
ok := errors.As(err, &structErr)
if !ok {
t.Errorf("Expected StructuredError, got %T", err)
} else if structErr.Code != gibidiutils.CodeResourceLimitTotalSize {
t.Errorf("Expected error code %s, got %s", gibidiutils.CodeResourceLimitTotalSize, structErr.Code)
assertStructuredError(t, err, shared.CodeResourceLimitTotalSize)
}
// TestResourceMonitor_MemoryLimitExceeded tests memory limit violation scenarios.
func TestResourceMonitorMemoryLimitExceeded(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set very low memory limit to try to force violations
viper.Set(shared.TestCfgResourceLimitsEnabled, true)
viper.Set("resourceLimits.hardMemoryLimitMB", 0.001) // 1KB - extremely low
rm := NewResourceMonitor()
defer rm.Close()
// Allocate large buffer to increase memory usage before check
largeBuffer := make([]byte, 10*1024*1024) // 10MB allocation
_ = largeBuffer[0] // Use the buffer to prevent optimization
// Check hard memory limit - might trigger if actual memory is high enough
err := rm.CheckHardMemoryLimit()
// Note: This test might not always fail since it depends on actual runtime memory
// But if it does fail, verify it's the correct error type
if err != nil {
validateMemoryLimitError(t, err)
t.Log("Successfully triggered memory limit violation")
} else {
t.Log("Memory limit check passed - actual memory usage may be within limits")
}
}
// TestResourceMonitor_MemoryLimitHandling tests the memory violation detection.
func TestResourceMonitorMemoryLimitHandling(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Enable resource limits with very small hard limit
viper.Set(shared.TestCfgResourceLimitsEnabled, true)
viper.Set("resourceLimits.hardMemoryLimitMB", 0.0001) // Very tiny limit
viper.Set("resourceLimits.enableGracefulDegradation", true)
rm := NewResourceMonitor()
defer rm.Close()
// Allocate more memory to increase chances of triggering limit
buffers := make([][]byte, 0, 100) // Pre-allocate capacity
for i := 0; i < 100; i++ {
buffer := make([]byte, 1024*1024) // 1MB each
buffers = append(buffers, buffer)
_ = buffer[0] // Use buffer
_ = buffers // Use the slice to prevent unused variable warning
// Check periodically
if i%10 == 0 {
err := rm.CheckHardMemoryLimit()
if err != nil {
// Successfully triggered memory limit
if !strings.Contains(err.Error(), "memory limit") {
t.Errorf("Expected error message to mention memory limit, got: %v", err)
}
t.Log("Successfully triggered memory limit handling")
return
}
}
}
t.Log("Could not trigger memory limit - actual memory usage may be lower than limit")
}
// TestResourceMonitorGracefulRecovery tests graceful recovery attempts.
func TestResourceMonitorGracefulRecovery(t *testing.T) {
testutil.ResetViperConfig(t, "")
// Set memory limits that will trigger recovery
viper.Set(shared.TestCfgResourceLimitsEnabled, true)
rm := NewResourceMonitor()
defer rm.Close()
// Force a deterministic 1-byte hard memory limit to trigger recovery
rm.hardMemoryLimitBytes = 1
// Process multiple files to accumulate memory usage
for i := 0; i < 3; i++ {
filePath := "/tmp/test" + string(rune('1'+i)) + ".txt"
fileSize := int64(400) // Each file is 400 bytes
// First few might pass, but eventually should trigger recovery mechanisms
err := rm.ValidateFileProcessing(filePath, fileSize)
if err != nil {
// Once we hit the limit, test that the error is appropriate
if !strings.Contains(err.Error(), "resource") && !strings.Contains(err.Error(), "limit") {
t.Errorf("Expected resource limit error, got: %v", err)
}
break
}
rm.RecordFileProcessed(fileSize)
}
}

23
fileproc/walker.go
View File

@@ -5,7 +5,7 @@ import (
"os"
"path/filepath"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// Walker defines an interface for scanning directories.
@@ -30,13 +30,16 @@ func NewProdWalker() *ProdWalker {
// Walk scans the given root directory recursively and returns a slice of file paths
// that are not ignored based on .gitignore/.ignore files, the configuration, or the default binary/image filter.
func (w *ProdWalker) Walk(root string) ([]string, error) {
absRoot, err := gibidiutils.GetAbsolutePath(root)
absRoot, err := shared.AbsolutePath(root)
if err != nil {
return nil, gibidiutils.WrapError(
err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSPathResolution,
return nil, shared.WrapError(
err,
shared.ErrorTypeFileSystem,
shared.CodeFSPathResolution,
"failed to resolve root path",
).WithFilePath(root)
}
return w.walkDir(absRoot, []ignoreRule{})
}
@@ -50,8 +53,10 @@ func (w *ProdWalker) walkDir(currentDir string, parentRules []ignoreRule) ([]str
entries, err := os.ReadDir(currentDir)
if err != nil {
return nil, gibidiutils.WrapError(
err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSAccess,
return nil, shared.WrapError(
err,
shared.ErrorTypeFileSystem,
shared.CodeFSAccess,
"failed to read directory",
).WithFilePath(currentDir)
}
@@ -69,8 +74,10 @@ func (w *ProdWalker) walkDir(currentDir string, parentRules []ignoreRule) ([]str
if entry.IsDir() {
subFiles, err := w.walkDir(fullPath, rules)
if err != nil {
return nil, gibidiutils.WrapError(
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingTraversal,
return nil, shared.WrapError(
err,
shared.ErrorTypeProcessing,
shared.CodeProcessingTraversal,
"failed to traverse subdirectory",
).WithFilePath(fullPath)
}

2
fileproc/walker_test.go
View File

@@ -61,8 +61,6 @@ func TestProdWalkerBinaryCheck(t *testing.T) {
// Reset FileTypeRegistry to ensure clean state
fileproc.ResetRegistryForTesting()
// Ensure cleanup runs even if test fails
t.Cleanup(fileproc.ResetRegistryForTesting)
// Run walker
w := fileproc.NewProdWalker()

123
fileproc/writer.go
View File

@@ -2,103 +2,66 @@
package fileproc
import (
"fmt"
"os"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// WriterConfig holds configuration for the writer.
type WriterConfig struct {
Format string
Prefix string
Suffix string
}
// startFormatWriter handles generic writer orchestration for any format.
// This eliminates code duplication across format-specific writer functions.
// Uses the FormatWriter interface defined in formats.go.
func startFormatWriter(
outFile *os.File,
writeCh <-chan WriteRequest,
done chan<- struct{},
prefix, suffix string,
writerFactory func(*os.File) FormatWriter,
) {
defer close(done)
// Validate checks if the WriterConfig is valid.
func (c WriterConfig) Validate() error {
if c.Format == "" {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationFormat,
"format cannot be empty",
"",
nil,
)
writer := writerFactory(outFile)
// Start writing
if err := writer.Start(prefix, suffix); err != nil {
shared.LogError("Failed to start writer", err)
return
}
switch c.Format {
case "markdown", "json", "yaml":
return nil
default:
context := map[string]any{
"format": c.Format,
// Process files
for req := range writeCh {
if err := writer.WriteFile(req); err != nil {
shared.LogError("Failed to write file", err)
}
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationFormat,
fmt.Sprintf("unsupported format: %s", c.Format),
"",
context,
)
}
// Close writer
if err := writer.Close(); err != nil {
shared.LogError("Failed to close writer", err)
}
}
// StartWriter writes the output in the specified format with memory optimization.
func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, config WriterConfig) {
// Validate config
if err := config.Validate(); err != nil {
gibidiutils.LogError("Invalid writer configuration", err)
close(done)
return
}
// Validate outFile is not nil
if outFile == nil {
err := gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOFileWrite,
"output file is nil",
"",
nil,
)
gibidiutils.LogError("Failed to write output", err)
close(done)
return
}
// Validate outFile is accessible
if _, err := outFile.Stat(); err != nil {
structErr := gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOFileWrite,
"failed to stat output file",
)
gibidiutils.LogError("Failed to validate output file", structErr)
close(done)
return
}
switch config.Format {
case "markdown":
startMarkdownWriter(outFile, writeCh, done, config.Prefix, config.Suffix)
case "json":
startJSONWriter(outFile, writeCh, done, config.Prefix, config.Suffix)
case "yaml":
startYAMLWriter(outFile, writeCh, done, config.Prefix, config.Suffix)
func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, format, prefix, suffix string) {
switch format {
case shared.FormatMarkdown:
startMarkdownWriter(outFile, writeCh, done, prefix, suffix)
case shared.FormatJSON:
startJSONWriter(outFile, writeCh, done, prefix, suffix)
case shared.FormatYAML:
startYAMLWriter(outFile, writeCh, done, prefix, suffix)
default:
context := map[string]interface{}{
"format": config.Format,
context := map[string]any{
"format": format,
}
err := gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationFormat,
fmt.Sprintf("unsupported format: %s", config.Format),
err := shared.NewStructuredError(
shared.ErrorTypeValidation,
shared.CodeValidationFormat,
"unsupported format: "+format,
"",
context,
)
gibidiutils.LogError("Failed to encode output", err)
shared.LogError("Failed to encode output", err)
close(done)
}
}

542
fileproc/writer_test.go
View File

@@ -2,17 +2,23 @@ package fileproc_test
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gopkg.in/yaml.v3"
"github.com/ivuorinen/gibidify/fileproc"
"github.com/ivuorinen/gibidify/shared"
)
func TestStartWriter_Formats(t *testing.T) {
func TestStartWriterFormats(t *testing.T) {
// Define table-driven test cases
tests := []struct {
name string
@@ -26,15 +32,17 @@ func TestStartWriter_Formats(t *testing.T) {
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
data := runWriterTest(t, tc.format)
if tc.expectError {
verifyErrorOutput(t, data)
} else {
verifyValidOutput(t, data, tc.format)
verifyPrefixSuffix(t, data)
}
})
t.Run(
tc.name, func(t *testing.T) {
data := runWriterTest(t, tc.format)
if tc.expectError {
verifyErrorOutput(t, data)
} else {
verifyValidOutput(t, data, tc.format)
verifyPrefixSuffix(t, data)
}
},
)
}
}
@@ -43,7 +51,7 @@ func runWriterTest(t *testing.T, format string) []byte {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "gibidify_test_output")
if err != nil {
t.Fatalf("Failed to create temp file: %v", err)
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
defer func() {
if closeErr := outFile.Close(); closeErr != nil {
@@ -59,25 +67,23 @@ func runWriterTest(t *testing.T, format string) []byte {
doneCh := make(chan struct{})
// Write a couple of sample requests
writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: "package main"}
writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: shared.LiteralPackageMain}
writeCh <- fileproc.WriteRequest{Path: "example.py", Content: "def foo(): pass"}
close(writeCh)
// Start the writer
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
fileproc.StartWriter(outFile, writeCh, doneCh, fileproc.WriterConfig{
Format: format,
Prefix: "PREFIX",
Suffix: "SUFFIX",
})
}()
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
})
// Wait until writer signals completion
wg.Wait()
<-doneCh // make sure all writes finished
select {
case <-doneCh: // make sure all writes finished
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
// Read output
data, err := os.ReadFile(outFile.Name())
@@ -115,6 +121,11 @@ func verifyValidOutput(t *testing.T, data []byte, format string) {
if !strings.Contains(content, "```") {
t.Error("Expected markdown code fences not found")
}
default:
// Unknown format - basic validation that we have content
if len(content) == 0 {
t.Errorf("Unexpected format %s with empty content", format)
}
}
}
@@ -129,3 +140,490 @@ func verifyPrefixSuffix(t *testing.T, data []byte) {
t.Errorf("Missing suffix in output: %s", data)
}
}
// verifyPrefixSuffixWith checks that output contains expected custom prefix and suffix.
func verifyPrefixSuffixWith(t *testing.T, data []byte, expectedPrefix, expectedSuffix string) {
t.Helper()
content := string(data)
if !strings.Contains(content, expectedPrefix) {
t.Errorf("Missing prefix '%s' in output: %s", expectedPrefix, data)
}
if !strings.Contains(content, expectedSuffix) {
t.Errorf("Missing suffix '%s' in output: %s", expectedSuffix, data)
}
}
// TestStartWriterStreamingFormats tests streaming functionality in all writers.
func TestStartWriterStreamingFormats(t *testing.T) {
tests := []struct {
name string
format string
content string
}{
{"JSON streaming", "json", strings.Repeat("line\n", 1000)},
{"YAML streaming", "yaml", strings.Repeat("data: value\n", 1000)},
{"Markdown streaming", "markdown", strings.Repeat("# Header\nContent\n", 1000)},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
data := runStreamingWriterTest(t, tc.format, tc.content)
// Verify output is not empty
if len(data) == 0 {
t.Error("Expected streaming output but got empty result")
}
// Format-specific validation
verifyValidOutput(t, data, tc.format)
verifyPrefixSuffixWith(t, data, "STREAM_PREFIX", "STREAM_SUFFIX")
// Verify content was written
content := string(data)
if !strings.Contains(content, shared.TestFileStreamTest) {
t.Error("Expected file path in streaming output")
}
},
)
}
}
// runStreamingWriterTest executes the writer with streaming content.
func runStreamingWriterTest(t *testing.T, format, content string) []byte {
t.Helper()
// Create temp file with content for streaming
contentFile, err := os.CreateTemp(t.TempDir(), "content_*.txt")
if err != nil {
t.Fatalf("Failed to create content file: %v", err)
}
defer func() {
if err := os.Remove(contentFile.Name()); err != nil {
t.Logf("Failed to remove content file: %v", err)
}
}()
if _, err := contentFile.WriteString(content); err != nil {
t.Fatalf("Failed to write content file: %v", err)
}
if err := contentFile.Close(); err != nil {
t.Fatalf("Failed to close content file: %v", err)
}
// Create output file
outFile, err := os.CreateTemp(t.TempDir(), "gibidify_stream_test_output")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
defer func() {
if closeErr := outFile.Close(); closeErr != nil {
t.Errorf("close temp file: %v", closeErr)
}
if removeErr := os.Remove(outFile.Name()); removeErr != nil {
t.Errorf("remove temp file: %v", removeErr)
}
}()
// Prepare channels with streaming request
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
// Create reader for streaming
reader, err := os.Open(contentFile.Name())
if err != nil {
t.Fatalf("Failed to open content file for reading: %v", err)
}
defer func() {
if err := reader.Close(); err != nil {
t.Logf("Failed to close reader: %v", err)
}
}()
// Write streaming request
writeCh <- fileproc.WriteRequest{
Path: shared.TestFileStreamTest,
Content: "", // Empty for streaming
IsStream: true,
Reader: reader,
}
close(writeCh)
// Start the writer
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "STREAM_PREFIX", "STREAM_SUFFIX")
})
// Wait until writer signals completion
wg.Wait()
select {
case <-doneCh:
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
// Read output
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Error reading output file: %v", err)
}
return data
}
// setupReadOnlyFile creates a read-only file for error testing.
func setupReadOnlyFile(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outPath := filepath.Join(t.TempDir(), "readonly_out")
outFile, err := os.Create(outPath)
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
// Close writable FD and reopen as read-only so writes will fail
_ = outFile.Close()
outFile, err = os.OpenFile(outPath, os.O_RDONLY, 0)
if err != nil {
t.Fatalf("Failed to reopen as read-only: %v", err)
}
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{
Path: shared.TestFileGo,
Content: shared.LiteralPackageMain,
}
close(writeCh)
return outFile, writeCh, doneCh
}
// setupStreamingError creates a streaming request with a failing reader.
func setupStreamingError(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "yaml_stream_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
pr, pw := io.Pipe()
if err := pw.CloseWithError(errors.New("simulated stream error")); err != nil {
t.Fatalf("failed to set pipe error: %v", err)
}
writeCh <- fileproc.WriteRequest{
Path: "stream_fail.yaml",
Content: "", // Empty for streaming
IsStream: true,
Reader: pr,
}
close(writeCh)
return outFile, writeCh, doneCh
}
// setupSpecialCharacters creates requests with special characters.
func setupSpecialCharacters(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "markdown_special_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
writeCh := make(chan fileproc.WriteRequest, 2)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{
Path: "special\ncharacters.md",
Content: "Content with\x00null bytes and\ttabs",
}
writeCh <- fileproc.WriteRequest{
Path: "empty.md",
Content: "",
}
close(writeCh)
return outFile, writeCh, doneCh
}
// runErrorHandlingTest runs a single error handling test.
func runErrorHandlingTest(
t *testing.T,
outFile *os.File,
writeCh chan fileproc.WriteRequest,
doneCh chan struct{},
format string,
expectEmpty bool,
) {
t.Helper()
defer func() {
if err := os.Remove(outFile.Name()); err != nil {
t.Logf("Failed to remove temp file: %v", err)
}
}()
defer func() {
if err := outFile.Close(); err != nil {
t.Logf("Failed to close temp file: %v", err)
}
}()
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
})
wg.Wait()
// Wait for doneCh with timeout to prevent test hangs
select {
case <-doneCh:
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
// Read output file and verify based on expectation
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Failed to read output file: %v", err)
}
if expectEmpty && len(data) != 0 {
t.Errorf("expected empty output on error, got %d bytes", len(data))
}
if !expectEmpty && len(data) == 0 {
t.Error("expected non-empty output, got empty")
}
}
// TestStartWriterErrorHandling tests error scenarios in writers.
func TestStartWriterErrorHandling(t *testing.T) {
tests := []struct {
name string
format string
setupError func(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{})
expectEmptyOutput bool
}{
{
name: "JSON writer with read-only file",
format: "json",
setupError: setupReadOnlyFile,
expectEmptyOutput: true,
},
{
name: "YAML writer with streaming error",
format: "yaml",
setupError: setupStreamingError,
expectEmptyOutput: false, // Partial writes are acceptable before streaming errors
},
{
name: "Markdown writer with special characters",
format: "markdown",
setupError: setupSpecialCharacters,
expectEmptyOutput: false,
},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
outFile, writeCh, doneCh := tc.setupError(t)
runErrorHandlingTest(t, outFile, writeCh, doneCh, tc.format, tc.expectEmptyOutput)
},
)
}
}
// setupCloseTest sets up files and channels for close testing.
func setupCloseTest(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "close_test_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
writeCh := make(chan fileproc.WriteRequest, 5)
doneCh := make(chan struct{})
for i := 0; i < 5; i++ {
writeCh <- fileproc.WriteRequest{
Path: fmt.Sprintf("file%d.txt", i),
Content: fmt.Sprintf("Content %d", i),
}
}
close(writeCh)
return outFile, writeCh, doneCh
}
// runCloseTest executes writer and validates output.
func runCloseTest(
t *testing.T,
outFile *os.File,
writeCh chan fileproc.WriteRequest,
doneCh chan struct{},
format string,
) {
t.Helper()
defer func() {
if err := os.Remove(outFile.Name()); err != nil {
t.Logf("Failed to remove temp file: %v", err)
}
}()
defer func() {
if err := outFile.Close(); err != nil {
t.Logf("Failed to close temp file: %v", err)
}
}()
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "TEST_PREFIX", "TEST_SUFFIX")
})
wg.Wait()
select {
case <-doneCh:
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Failed to read output file: %v", err)
}
if len(data) == 0 {
t.Error("Expected non-empty output file")
}
verifyPrefixSuffixWith(t, data, "TEST_PREFIX", "TEST_SUFFIX")
}
// TestStartWriterWriterCloseErrors tests error handling during writer close operations.
func TestStartWriterWriterCloseErrors(t *testing.T) {
tests := []struct {
name string
format string
}{
{"JSON close handling", "json"},
{"YAML close handling", "yaml"},
{"Markdown close handling", "markdown"},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
outFile, writeCh, doneCh := setupCloseTest(t)
runCloseTest(t, outFile, writeCh, doneCh, tc.format)
},
)
}
}
// Benchmarks for writer performance
// BenchmarkStartWriter benchmarks basic writer operations across formats.
func BenchmarkStartWriter(b *testing.B) {
formats := []string{"json", "yaml", "markdown"}
for _, format := range formats {
b.Run(format, func(b *testing.B) {
for b.Loop() {
outFile, err := os.CreateTemp(b.TempDir(), "bench_output_*")
if err != nil {
b.Fatalf("Failed to create temp file: %v", err)
}
writeCh := make(chan fileproc.WriteRequest, 2)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: shared.LiteralPackageMain}
writeCh <- fileproc.WriteRequest{Path: "example.py", Content: "def foo(): pass"}
close(writeCh)
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
<-doneCh
_ = outFile.Close()
}
})
}
}
// benchStreamingIteration runs a single streaming benchmark iteration.
func benchStreamingIteration(b *testing.B, format, content string) {
b.Helper()
contentFile := createBenchContentFile(b, content)
defer func() { _ = os.Remove(contentFile) }()
reader, err := os.Open(contentFile)
if err != nil {
b.Fatalf("Failed to open content file: %v", err)
}
defer func() { _ = reader.Close() }()
outFile, err := os.CreateTemp(b.TempDir(), "bench_stream_output_*")
if err != nil {
b.Fatalf("Failed to create output file: %v", err)
}
defer func() { _ = outFile.Close() }()
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{
Path: shared.TestFileStreamTest,
Content: "",
IsStream: true,
Reader: reader,
}
close(writeCh)
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
<-doneCh
}
// createBenchContentFile creates a temp file with content for benchmarks.
func createBenchContentFile(b *testing.B, content string) string {
b.Helper()
contentFile, err := os.CreateTemp(b.TempDir(), "content_*")
if err != nil {
b.Fatalf("Failed to create content file: %v", err)
}
if _, err := contentFile.WriteString(content); err != nil {
b.Fatalf("Failed to write content: %v", err)
}
if err := contentFile.Close(); err != nil {
b.Fatalf("Failed to close content file: %v", err)
}
return contentFile.Name()
}
// BenchmarkStartWriterStreaming benchmarks streaming writer operations across formats.
func BenchmarkStartWriterStreaming(b *testing.B) {
formats := []string{"json", "yaml", "markdown"}
content := strings.Repeat("line content\n", 1000)
for _, format := range formats {
b.Run(format, func(b *testing.B) {
for b.Loop() {
benchStreamingIteration(b, format, content)
}
})
}
}

263
fileproc/yaml_writer.go
View File

@@ -1,14 +1,12 @@
// Package fileproc handles file processing, collection, and output formatting.
package fileproc
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/ivuorinen/gibidify/gibidiutils"
"github.com/ivuorinen/gibidify/shared"
)
// YAMLWriter handles YAML format output with streaming support.
@@ -21,152 +19,18 @@ func NewYAMLWriter(outFile *os.File) *YAMLWriter {
return &YAMLWriter{outFile: outFile}
}
const (
maxPathLength = 4096 // Maximum total path length
maxFilenameLength = 255 // Maximum individual filename component length
)
// validatePathComponents validates individual path components for security issues.
func validatePathComponents(trimmed, cleaned string, components []string) error {
for i, component := range components {
// Reject path components that are exactly ".." (path traversal)
if component == ".." {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path traversal not allowed",
trimmed,
map[string]any{
"path": trimmed,
"cleaned": cleaned,
"invalid_component": component,
"component_index": i,
},
)
}
// Reject empty components (e.g., from "foo//bar")
if component == "" && i > 0 && i < len(components)-1 {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path contains empty component",
trimmed,
map[string]any{
"path": trimmed,
"cleaned": cleaned,
"component_index": i,
},
)
}
// Enforce maximum filename length for each component
if len(component) > maxFilenameLength {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path component exceeds maximum length",
trimmed,
map[string]any{
"component": component,
"component_length": len(component),
"max_length": maxFilenameLength,
"component_index": i,
},
)
}
}
return nil
}
// validatePath validates and sanitizes a file path for safe output.
// It rejects absolute paths, path traversal attempts, empty paths, and overly long paths.
func validatePath(path string) error {
// Reject empty paths
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationRequired,
"file path cannot be empty",
"",
nil,
)
}
// Enforce maximum path length to prevent resource abuse
if len(trimmed) > maxPathLength {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path exceeds maximum length",
trimmed,
map[string]any{
"path_length": len(trimmed),
"max_length": maxPathLength,
},
)
}
// Reject absolute paths
if filepath.IsAbs(trimmed) {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"absolute paths are not allowed",
trimmed,
map[string]any{"path": trimmed},
)
}
// Validate original trimmed path components before cleaning
origComponents := strings.Split(filepath.ToSlash(trimmed), "/")
for _, comp := range origComponents {
if comp == "" || comp == "." || comp == ".." {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"invalid or traversal path component in original path",
trimmed,
map[string]any{"path": trimmed, "component": comp},
)
}
}
// Clean the path to normalize it
cleaned := filepath.Clean(trimmed)
// After cleaning, ensure it's still relative and doesn't start with /
if filepath.IsAbs(cleaned) || strings.HasPrefix(cleaned, "/") {
return gibidiutils.NewStructuredError(
gibidiutils.ErrorTypeValidation,
gibidiutils.CodeValidationPath,
"path must be relative",
trimmed,
map[string]any{"path": trimmed, "cleaned": cleaned},
)
}
// Split into components and validate each one
// Use ToSlash to normalize for cross-platform validation
components := strings.Split(filepath.ToSlash(cleaned), "/")
return validatePathComponents(trimmed, cleaned, components)
}
// Start writes the YAML header.
func (w *YAMLWriter) Start(prefix, suffix string) error {
// Write YAML header
if _, err := fmt.Fprintf(
w.outFile, "prefix: %s\nsuffix: %s\nfiles:\n",
gibidiutils.EscapeForYAML(prefix), gibidiutils.EscapeForYAML(suffix),
w.outFile,
"prefix: %s\nsuffix: %s\nfiles:\n",
shared.EscapeForYAML(prefix),
shared.EscapeForYAML(suffix),
); err != nil {
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOWrite,
"failed to write YAML header",
)
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "failed to write YAML header")
}
return nil
}
@@ -175,6 +39,7 @@ func (w *YAMLWriter) WriteFile(req WriteRequest) error {
if req.IsStream {
return w.writeStreaming(req)
}
return w.writeInline(req)
}
@@ -185,45 +50,39 @@ func (w *YAMLWriter) Close() error {
// writeStreaming writes a large file as YAML in streaming chunks.
func (w *YAMLWriter) writeStreaming(req WriteRequest) error {
// Validate path before using it
if err := validatePath(req.Path); err != nil {
return err
}
// Check for nil reader
if req.Reader == nil {
return gibidiutils.WrapError(
nil, gibidiutils.ErrorTypeValidation, gibidiutils.CodeValidationRequired,
"nil reader in write request",
).WithFilePath(req.Path)
}
defer gibidiutils.SafeCloseReader(req.Reader, req.Path)
defer shared.SafeCloseReader(req.Reader, req.Path)
language := detectLanguage(req.Path)
// Write YAML file entry start
if _, err := fmt.Fprintf(
w.outFile, " - path: %s\n language: %s\n content: |\n",
gibidiutils.EscapeForYAML(req.Path), language,
w.outFile,
shared.YAMLFmtFileEntry,
shared.EscapeForYAML(req.Path),
language,
); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write YAML file start",
).WithFilePath(req.Path)
}
// Stream content with YAML indentation
return w.streamYAMLContent(req.Reader, req.Path)
if err := shared.StreamLines(
req.Reader, w.outFile, req.Path, func(line string) string {
return " " + line
},
); err != nil {
return shared.WrapError(err, shared.ErrorTypeIO, shared.CodeIOWrite, "streaming YAML content")
}
return nil
}
// writeInline writes a small file directly as YAML.
func (w *YAMLWriter) writeInline(req WriteRequest) error {
// Validate path before using it
if err := validatePath(req.Path); err != nil {
return err
}
language := detectLanguage(req.Path)
fileData := FileData{
Path: req.Path,
@@ -233,11 +92,15 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error {
// Write YAML entry
if _, err := fmt.Fprintf(
w.outFile, " - path: %s\n language: %s\n content: |\n",
gibidiutils.EscapeForYAML(fileData.Path), fileData.Language,
w.outFile,
shared.YAMLFmtFileEntry,
shared.EscapeForYAML(fileData.Path),
fileData.Language,
); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write YAML entry start",
).WithFilePath(req.Path)
}
@@ -246,8 +109,10 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error {
lines := strings.Split(fileData.Content, "\n")
for _, line := range lines {
if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
return shared.WrapError(
err,
shared.ErrorTypeIO,
shared.CodeIOWrite,
"failed to write YAML content line",
).WithFilePath(req.Path)
}
@@ -256,53 +121,9 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error {
return nil
}
// streamYAMLContent streams content with YAML indentation.
func (w *YAMLWriter) streamYAMLContent(reader io.Reader, path string) error {
scanner := bufio.NewScanner(reader)
// Increase buffer size to handle long lines (up to 10MB per line)
buf := make([]byte, 0, 64*1024)
scanner.Buffer(buf, 10*1024*1024)
for scanner.Scan() {
line := scanner.Text()
if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
"failed to write YAML line",
).WithFilePath(path)
}
}
if err := scanner.Err(); err != nil {
return gibidiutils.WrapError(
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOFileRead,
"failed to scan YAML content",
).WithFilePath(path)
}
return nil
}
// startYAMLWriter handles YAML format output with streaming support.
func startYAMLWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
defer close(done)
writer := NewYAMLWriter(outFile)
// Start writing
if err := writer.Start(prefix, suffix); err != nil {
gibidiutils.LogError("Failed to write YAML header", err)
return
}
// Process files
for req := range writeCh {
if err := writer.WriteFile(req); err != nil {
gibidiutils.LogError("Failed to write YAML file", err)
}
}
// Close writer
if err := writer.Close(); err != nil {
gibidiutils.LogError("Failed to write YAML end", err)
}
startFormatWriter(outFile, writeCh, done, prefix, suffix, func(f *os.File) FormatWriter {
return NewYAMLWriter(f)
})
}