chore: modernize workflows, security scanning, and linting configuration (#50)

* build: update Go 1.25, CI workflows, and build tooling - Upgrade to Go 1.25 - Add benchmark targets to Makefile - Implement parallel gosec execution - Lock tool versions for reproducibility - Add shellcheck directives to scripts - Update CI workflows with improved caching * refactor: migrate from golangci-lint to revive - Replace golangci-lint with revive for linting - Configure comprehensive revive rules - Fix all EditorConfig violations - Add yamllint and yamlfmt support - Remove deprecated .golangci.yml * refactor: rename utils to shared and deduplicate code - Rename utils package to shared - Add shared constants package - Deduplicate constants across packages - Address CodeRabbit review feedback * fix: resolve SonarQube issues and add safety guards - Fix all 73 SonarQube OPEN issues - Add nil guards for resourceMonitor, backpressure, metricsCollector - Implement io.Closer for headerFileReader - Propagate errors from processing helpers - Add metrics and templates packages - Improve error handling across codebase * test: improve test infrastructure and coverage - Add benchmarks for cli, fileproc, metrics - Improve test coverage for cli, fileproc, config - Refactor tests with helper functions - Add shared test constants - Fix test function naming conventions - Reduce cognitive complexity in benchmark tests * docs: update documentation and configuration examples - Update CLAUDE.md with current project state - Refresh README with new features - Add usage and configuration examples - Add SonarQube project configuration - Consolidate config.example.yaml * fix: resolve shellcheck warnings in scripts - Use ./*.go instead of *.go to prevent dash-prefixed filenames from being interpreted as options (SC2035) - Remove unreachable return statement after exit (SC2317) - Remove obsolete gibidiutils/ directory reference * chore(deps): upgrade go dependencies * chore(lint): megalinter fixes * fix: improve test coverage and fix file descriptor leaks - Add defer r.Close() to fix pipe file descriptor leaks in benchmark tests - Refactor TestProcessorConfigureFileTypes with helper functions and assertions - Refactor TestProcessorLogFinalStats with output capture and keyword verification - Use shared constants instead of literal strings (TestFilePNG, FormatMarkdown, etc.) - Reduce cognitive complexity by extracting helper functions * fix: align test comments with function names Remove underscores from test comments to match actual function names: - benchmark/benchmark_test.go (2 fixes) - fileproc/filetypes_config_test.go (4 fixes) - fileproc/filetypes_registry_test.go (6 fixes) - fileproc/processor_test.go (6 fixes) - fileproc/resource_monitor_types_test.go (4 fixes) - fileproc/writer_test.go (3 fixes) * fix: various test improvements and bug fixes - Remove duplicate maxCacheSize check in filetypes_registry_test.go - Shorten long comment in processor_test.go to stay under 120 chars - Remove flaky time.Sleep in collector_test.go, use >= 0 assertion - Close pipe reader in benchmark_test.go to fix file descriptor leak - Use ContinueOnError in flags_test.go to match ResetFlags behavior - Add nil check for p.ui in processor_workers.go before UpdateProgress - Fix resource_monitor_validation_test.go by setting hardMemoryLimitBytes directly * chore(yaml): add missing document start markers Add --- document start to YAML files to satisfy yamllint: - .github/workflows/codeql.yml - .github/workflows/build-test-publish.yml - .github/workflows/security.yml - .github/actions/setup/action.yml * fix: guard nil resourceMonitor and fix test deadlock - Guard resourceMonitor before CreateFileProcessingContext call - Add ui.UpdateProgress on emergency stop and path error returns - Fix potential deadlock in TestProcessFile using wg.Go with defer close
2026-03-01 00:55:29 +00:00 · 2025-12-10 19:07:11 +02:00
parent ea4a39a360
commit 95b7ef6dd3
149 changed files with 22990 additions and 8976 deletions
--- a/fileproc/writer_test.go
+++ b/fileproc/writer_test.go
@@ -2,17 +2,23 @@ package fileproc_test

 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
+	"time"

 	"gopkg.in/yaml.v3"

 	"github.com/ivuorinen/gibidify/fileproc"
+	"github.com/ivuorinen/gibidify/shared"
 )

-func TestStartWriter_Formats(t *testing.T) {
+func TestStartWriterFormats(t *testing.T) {
 	// Define table-driven test cases
 	tests := []struct {
 		name        string
@@ -26,15 +32,17 @@ func TestStartWriter_Formats(t *testing.T) {
 	}

 	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			data := runWriterTest(t, tc.format)
-			if tc.expectError {
-				verifyErrorOutput(t, data)
-			} else {
-				verifyValidOutput(t, data, tc.format)
-				verifyPrefixSuffix(t, data)
-			}
-		})
+		t.Run(
+			tc.name, func(t *testing.T) {
+				data := runWriterTest(t, tc.format)
+				if tc.expectError {
+					verifyErrorOutput(t, data)
+				} else {
+					verifyValidOutput(t, data, tc.format)
+					verifyPrefixSuffix(t, data)
+				}
+			},
+		)
 	}
 }

@@ -43,7 +51,7 @@ func runWriterTest(t *testing.T, format string) []byte {
 	t.Helper()
 	outFile, err := os.CreateTemp(t.TempDir(), "gibidify_test_output")
 	if err != nil {
-		t.Fatalf("Failed to create temp file: %v", err)
+		t.Fatalf(shared.TestMsgFailedToCreateFile, err)
 	}
 	defer func() {
 		if closeErr := outFile.Close(); closeErr != nil {
@@ -59,25 +67,23 @@ func runWriterTest(t *testing.T, format string) []byte {
 	doneCh := make(chan struct{})

 	// Write a couple of sample requests
-	writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: "package main"}
+	writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: shared.LiteralPackageMain}
 	writeCh <- fileproc.WriteRequest{Path: "example.py", Content: "def foo(): pass"}
 	close(writeCh)

 	// Start the writer
 	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		fileproc.StartWriter(outFile, writeCh, doneCh, fileproc.WriterConfig{
-			Format: format,
-			Prefix: "PREFIX",
-			Suffix: "SUFFIX",
-		})
-	}()
+	wg.Go(func() {
+		fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
+	})

 	// Wait until writer signals completion
 	wg.Wait()
-	<-doneCh // make sure all writes finished
+	select {
+	case <-doneCh: // make sure all writes finished
+	case <-time.After(3 * time.Second):
+		t.Fatal(shared.TestMsgTimeoutWriterCompletion)
+	}

 	// Read output
 	data, err := os.ReadFile(outFile.Name())
@@ -115,6 +121,11 @@ func verifyValidOutput(t *testing.T, data []byte, format string) {
 		if !strings.Contains(content, "```") {
 			t.Error("Expected markdown code fences not found")
 		}
+	default:
+		// Unknown format - basic validation that we have content
+		if len(content) == 0 {
+			t.Errorf("Unexpected format %s with empty content", format)
+		}
 	}
 }

@@ -129,3 +140,490 @@ func verifyPrefixSuffix(t *testing.T, data []byte) {
 		t.Errorf("Missing suffix in output: %s", data)
 	}
 }
+
+// verifyPrefixSuffixWith checks that output contains expected custom prefix and suffix.
+func verifyPrefixSuffixWith(t *testing.T, data []byte, expectedPrefix, expectedSuffix string) {
+	t.Helper()
+	content := string(data)
+	if !strings.Contains(content, expectedPrefix) {
+		t.Errorf("Missing prefix '%s' in output: %s", expectedPrefix, data)
+	}
+	if !strings.Contains(content, expectedSuffix) {
+		t.Errorf("Missing suffix '%s' in output: %s", expectedSuffix, data)
+	}
+}
+
+// TestStartWriterStreamingFormats tests streaming functionality in all writers.
+func TestStartWriterStreamingFormats(t *testing.T) {
+	tests := []struct {
+		name    string
+		format  string
+		content string
+	}{
+		{"JSON streaming", "json", strings.Repeat("line\n", 1000)},
+		{"YAML streaming", "yaml", strings.Repeat("data: value\n", 1000)},
+		{"Markdown streaming", "markdown", strings.Repeat("# Header\nContent\n", 1000)},
+	}
+
+	for _, tc := range tests {
+		t.Run(
+			tc.name, func(t *testing.T) {
+				data := runStreamingWriterTest(t, tc.format, tc.content)
+
+				// Verify output is not empty
+				if len(data) == 0 {
+					t.Error("Expected streaming output but got empty result")
+				}
+
+				// Format-specific validation
+				verifyValidOutput(t, data, tc.format)
+				verifyPrefixSuffixWith(t, data, "STREAM_PREFIX", "STREAM_SUFFIX")
+
+				// Verify content was written
+				content := string(data)
+				if !strings.Contains(content, shared.TestFileStreamTest) {
+					t.Error("Expected file path in streaming output")
+				}
+			},
+		)
+	}
+}
+
+// runStreamingWriterTest executes the writer with streaming content.
+func runStreamingWriterTest(t *testing.T, format, content string) []byte {
+	t.Helper()
+
+	// Create temp file with content for streaming
+	contentFile, err := os.CreateTemp(t.TempDir(), "content_*.txt")
+	if err != nil {
+		t.Fatalf("Failed to create content file: %v", err)
+	}
+	defer func() {
+		if err := os.Remove(contentFile.Name()); err != nil {
+			t.Logf("Failed to remove content file: %v", err)
+		}
+	}()
+
+	if _, err := contentFile.WriteString(content); err != nil {
+		t.Fatalf("Failed to write content file: %v", err)
+	}
+	if err := contentFile.Close(); err != nil {
+		t.Fatalf("Failed to close content file: %v", err)
+	}
+
+	// Create output file
+	outFile, err := os.CreateTemp(t.TempDir(), "gibidify_stream_test_output")
+	if err != nil {
+		t.Fatalf(shared.TestMsgFailedToCreateFile, err)
+	}
+	defer func() {
+		if closeErr := outFile.Close(); closeErr != nil {
+			t.Errorf("close temp file: %v", closeErr)
+		}
+		if removeErr := os.Remove(outFile.Name()); removeErr != nil {
+			t.Errorf("remove temp file: %v", removeErr)
+		}
+	}()
+
+	// Prepare channels with streaming request
+	writeCh := make(chan fileproc.WriteRequest, 1)
+	doneCh := make(chan struct{})
+
+	// Create reader for streaming
+	reader, err := os.Open(contentFile.Name())
+	if err != nil {
+		t.Fatalf("Failed to open content file for reading: %v", err)
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			t.Logf("Failed to close reader: %v", err)
+		}
+	}()
+
+	// Write streaming request
+	writeCh <- fileproc.WriteRequest{
+		Path:     shared.TestFileStreamTest,
+		Content:  "", // Empty for streaming
+		IsStream: true,
+		Reader:   reader,
+	}
+	close(writeCh)
+
+	// Start the writer
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		fileproc.StartWriter(outFile, writeCh, doneCh, format, "STREAM_PREFIX", "STREAM_SUFFIX")
+	})
+
+	// Wait until writer signals completion
+	wg.Wait()
+	select {
+	case <-doneCh:
+	case <-time.After(3 * time.Second):
+		t.Fatal(shared.TestMsgTimeoutWriterCompletion)
+	}
+
+	// Read output
+	data, err := os.ReadFile(outFile.Name())
+	if err != nil {
+		t.Fatalf("Error reading output file: %v", err)
+	}
+
+	return data
+}
+
+// setupReadOnlyFile creates a read-only file for error testing.
+func setupReadOnlyFile(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
+	t.Helper()
+
+	outPath := filepath.Join(t.TempDir(), "readonly_out")
+	outFile, err := os.Create(outPath)
+	if err != nil {
+		t.Fatalf(shared.TestMsgFailedToCreateFile, err)
+	}
+
+	// Close writable FD and reopen as read-only so writes will fail
+	_ = outFile.Close()
+	outFile, err = os.OpenFile(outPath, os.O_RDONLY, 0)
+	if err != nil {
+		t.Fatalf("Failed to reopen as read-only: %v", err)
+	}
+
+	writeCh := make(chan fileproc.WriteRequest, 1)
+	doneCh := make(chan struct{})
+
+	writeCh <- fileproc.WriteRequest{
+		Path:    shared.TestFileGo,
+		Content: shared.LiteralPackageMain,
+	}
+	close(writeCh)
+
+	return outFile, writeCh, doneCh
+}
+
+// setupStreamingError creates a streaming request with a failing reader.
+func setupStreamingError(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
+	t.Helper()
+
+	outFile, err := os.CreateTemp(t.TempDir(), "yaml_stream_*")
+	if err != nil {
+		t.Fatalf(shared.TestMsgFailedToCreateFile, err)
+	}
+
+	writeCh := make(chan fileproc.WriteRequest, 1)
+	doneCh := make(chan struct{})
+
+	pr, pw := io.Pipe()
+	if err := pw.CloseWithError(errors.New("simulated stream error")); err != nil {
+		t.Fatalf("failed to set pipe error: %v", err)
+	}
+
+	writeCh <- fileproc.WriteRequest{
+		Path:     "stream_fail.yaml",
+		Content:  "", // Empty for streaming
+		IsStream: true,
+		Reader:   pr,
+	}
+	close(writeCh)
+
+	return outFile, writeCh, doneCh
+}
+
+// setupSpecialCharacters creates requests with special characters.
+func setupSpecialCharacters(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
+	t.Helper()
+
+	outFile, err := os.CreateTemp(t.TempDir(), "markdown_special_*")
+	if err != nil {
+		t.Fatalf(shared.TestMsgFailedToCreateFile, err)
+	}
+
+	writeCh := make(chan fileproc.WriteRequest, 2)
+	doneCh := make(chan struct{})
+
+	writeCh <- fileproc.WriteRequest{
+		Path:    "special\ncharacters.md",
+		Content: "Content with\x00null bytes and\ttabs",
+	}
+
+	writeCh <- fileproc.WriteRequest{
+		Path:    "empty.md",
+		Content: "",
+	}
+	close(writeCh)
+
+	return outFile, writeCh, doneCh
+}
+
+// runErrorHandlingTest runs a single error handling test.
+func runErrorHandlingTest(
+	t *testing.T,
+	outFile *os.File,
+	writeCh chan fileproc.WriteRequest,
+	doneCh chan struct{},
+	format string,
+	expectEmpty bool,
+) {
+	t.Helper()
+
+	defer func() {
+		if err := os.Remove(outFile.Name()); err != nil {
+			t.Logf("Failed to remove temp file: %v", err)
+		}
+	}()
+	defer func() {
+		if err := outFile.Close(); err != nil {
+			t.Logf("Failed to close temp file: %v", err)
+		}
+	}()
+
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
+	})
+
+	wg.Wait()
+
+	// Wait for doneCh with timeout to prevent test hangs
+	select {
+	case <-doneCh:
+	case <-time.After(3 * time.Second):
+		t.Fatal(shared.TestMsgTimeoutWriterCompletion)
+	}
+
+	// Read output file and verify based on expectation
+	data, err := os.ReadFile(outFile.Name())
+	if err != nil {
+		t.Fatalf("Failed to read output file: %v", err)
+	}
+
+	if expectEmpty && len(data) != 0 {
+		t.Errorf("expected empty output on error, got %d bytes", len(data))
+	}
+	if !expectEmpty && len(data) == 0 {
+		t.Error("expected non-empty output, got empty")
+	}
+}
+
+// TestStartWriterErrorHandling tests error scenarios in writers.
+func TestStartWriterErrorHandling(t *testing.T) {
+	tests := []struct {
+		name              string
+		format            string
+		setupError        func(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{})
+		expectEmptyOutput bool
+	}{
+		{
+			name:              "JSON writer with read-only file",
+			format:            "json",
+			setupError:        setupReadOnlyFile,
+			expectEmptyOutput: true,
+		},
+		{
+			name:              "YAML writer with streaming error",
+			format:            "yaml",
+			setupError:        setupStreamingError,
+			expectEmptyOutput: false, // Partial writes are acceptable before streaming errors
+		},
+		{
+			name:              "Markdown writer with special characters",
+			format:            "markdown",
+			setupError:        setupSpecialCharacters,
+			expectEmptyOutput: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(
+			tc.name, func(t *testing.T) {
+				outFile, writeCh, doneCh := tc.setupError(t)
+				runErrorHandlingTest(t, outFile, writeCh, doneCh, tc.format, tc.expectEmptyOutput)
+			},
+		)
+	}
+}
+
+// setupCloseTest sets up files and channels for close testing.
+func setupCloseTest(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
+	t.Helper()
+
+	outFile, err := os.CreateTemp(t.TempDir(), "close_test_*")
+	if err != nil {
+		t.Fatalf(shared.TestMsgFailedToCreateFile, err)
+	}
+
+	writeCh := make(chan fileproc.WriteRequest, 5)
+	doneCh := make(chan struct{})
+
+	for i := 0; i < 5; i++ {
+		writeCh <- fileproc.WriteRequest{
+			Path:    fmt.Sprintf("file%d.txt", i),
+			Content: fmt.Sprintf("Content %d", i),
+		}
+	}
+	close(writeCh)
+
+	return outFile, writeCh, doneCh
+}
+
+// runCloseTest executes writer and validates output.
+func runCloseTest(
+	t *testing.T,
+	outFile *os.File,
+	writeCh chan fileproc.WriteRequest,
+	doneCh chan struct{},
+	format string,
+) {
+	t.Helper()
+
+	defer func() {
+		if err := os.Remove(outFile.Name()); err != nil {
+			t.Logf("Failed to remove temp file: %v", err)
+		}
+	}()
+	defer func() {
+		if err := outFile.Close(); err != nil {
+			t.Logf("Failed to close temp file: %v", err)
+		}
+	}()
+
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		fileproc.StartWriter(outFile, writeCh, doneCh, format, "TEST_PREFIX", "TEST_SUFFIX")
+	})
+
+	wg.Wait()
+	select {
+	case <-doneCh:
+	case <-time.After(3 * time.Second):
+		t.Fatal(shared.TestMsgTimeoutWriterCompletion)
+	}
+
+	data, err := os.ReadFile(outFile.Name())
+	if err != nil {
+		t.Fatalf("Failed to read output file: %v", err)
+	}
+
+	if len(data) == 0 {
+		t.Error("Expected non-empty output file")
+	}
+
+	verifyPrefixSuffixWith(t, data, "TEST_PREFIX", "TEST_SUFFIX")
+}
+
+// TestStartWriterWriterCloseErrors tests error handling during writer close operations.
+func TestStartWriterWriterCloseErrors(t *testing.T) {
+	tests := []struct {
+		name   string
+		format string
+	}{
+		{"JSON close handling", "json"},
+		{"YAML close handling", "yaml"},
+		{"Markdown close handling", "markdown"},
+	}
+
+	for _, tc := range tests {
+		t.Run(
+			tc.name, func(t *testing.T) {
+				outFile, writeCh, doneCh := setupCloseTest(t)
+				runCloseTest(t, outFile, writeCh, doneCh, tc.format)
+			},
+		)
+	}
+}
+
+// Benchmarks for writer performance
+
+// BenchmarkStartWriter benchmarks basic writer operations across formats.
+func BenchmarkStartWriter(b *testing.B) {
+	formats := []string{"json", "yaml", "markdown"}
+
+	for _, format := range formats {
+		b.Run(format, func(b *testing.B) {
+			for b.Loop() {
+				outFile, err := os.CreateTemp(b.TempDir(), "bench_output_*")
+				if err != nil {
+					b.Fatalf("Failed to create temp file: %v", err)
+				}
+
+				writeCh := make(chan fileproc.WriteRequest, 2)
+				doneCh := make(chan struct{})
+
+				writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: shared.LiteralPackageMain}
+				writeCh <- fileproc.WriteRequest{Path: "example.py", Content: "def foo(): pass"}
+				close(writeCh)
+
+				fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
+				<-doneCh
+
+				_ = outFile.Close()
+			}
+		})
+	}
+}
+
+// benchStreamingIteration runs a single streaming benchmark iteration.
+func benchStreamingIteration(b *testing.B, format, content string) {
+	b.Helper()
+
+	contentFile := createBenchContentFile(b, content)
+	defer func() { _ = os.Remove(contentFile) }()
+
+	reader, err := os.Open(contentFile)
+	if err != nil {
+		b.Fatalf("Failed to open content file: %v", err)
+	}
+	defer func() { _ = reader.Close() }()
+
+	outFile, err := os.CreateTemp(b.TempDir(), "bench_stream_output_*")
+	if err != nil {
+		b.Fatalf("Failed to create output file: %v", err)
+	}
+	defer func() { _ = outFile.Close() }()
+
+	writeCh := make(chan fileproc.WriteRequest, 1)
+	doneCh := make(chan struct{})
+
+	writeCh <- fileproc.WriteRequest{
+		Path:     shared.TestFileStreamTest,
+		Content:  "",
+		IsStream: true,
+		Reader:   reader,
+	}
+	close(writeCh)
+
+	fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
+	<-doneCh
+}
+
+// createBenchContentFile creates a temp file with content for benchmarks.
+func createBenchContentFile(b *testing.B, content string) string {
+	b.Helper()
+
+	contentFile, err := os.CreateTemp(b.TempDir(), "content_*")
+	if err != nil {
+		b.Fatalf("Failed to create content file: %v", err)
+	}
+	if _, err := contentFile.WriteString(content); err != nil {
+		b.Fatalf("Failed to write content: %v", err)
+	}
+	if err := contentFile.Close(); err != nil {
+		b.Fatalf("Failed to close content file: %v", err)
+	}
+
+	return contentFile.Name()
+}
+
+// BenchmarkStartWriterStreaming benchmarks streaming writer operations across formats.
+func BenchmarkStartWriterStreaming(b *testing.B) {
+	formats := []string{"json", "yaml", "markdown"}
+	content := strings.Repeat("line content\n", 1000)
+
+	for _, format := range formats {
+		b.Run(format, func(b *testing.B) {
+			for b.Loop() {
+				benchStreamingIteration(b, format, content)
+			}
+		})
+	}
+}