Files
gibidify/fileproc/writer_test.go
Ismo Vuorinen 95b7ef6dd3 chore: modernize workflows, security scanning, and linting configuration (#50)
* build: update Go 1.25, CI workflows, and build tooling

- Upgrade to Go 1.25
- Add benchmark targets to Makefile
- Implement parallel gosec execution
- Lock tool versions for reproducibility
- Add shellcheck directives to scripts
- Update CI workflows with improved caching

* refactor: migrate from golangci-lint to revive

- Replace golangci-lint with revive for linting
- Configure comprehensive revive rules
- Fix all EditorConfig violations
- Add yamllint and yamlfmt support
- Remove deprecated .golangci.yml

* refactor: rename utils to shared and deduplicate code

- Rename utils package to shared
- Add shared constants package
- Deduplicate constants across packages
- Address CodeRabbit review feedback

* fix: resolve SonarQube issues and add safety guards

- Fix all 73 SonarQube OPEN issues
- Add nil guards for resourceMonitor, backpressure, metricsCollector
- Implement io.Closer for headerFileReader
- Propagate errors from processing helpers
- Add metrics and templates packages
- Improve error handling across codebase

* test: improve test infrastructure and coverage

- Add benchmarks for cli, fileproc, metrics
- Improve test coverage for cli, fileproc, config
- Refactor tests with helper functions
- Add shared test constants
- Fix test function naming conventions
- Reduce cognitive complexity in benchmark tests

* docs: update documentation and configuration examples

- Update CLAUDE.md with current project state
- Refresh README with new features
- Add usage and configuration examples
- Add SonarQube project configuration
- Consolidate config.example.yaml

* fix: resolve shellcheck warnings in scripts

- Use ./*.go instead of *.go to prevent dash-prefixed filenames
  from being interpreted as options (SC2035)
- Remove unreachable return statement after exit (SC2317)
- Remove obsolete gibidiutils/ directory reference

* chore(deps): upgrade go dependencies

* chore(lint): megalinter fixes

* fix: improve test coverage and fix file descriptor leaks

- Add defer r.Close() to fix pipe file descriptor leaks in benchmark tests
- Refactor TestProcessorConfigureFileTypes with helper functions and assertions
- Refactor TestProcessorLogFinalStats with output capture and keyword verification
- Use shared constants instead of literal strings (TestFilePNG, FormatMarkdown, etc.)
- Reduce cognitive complexity by extracting helper functions

* fix: align test comments with function names

Remove underscores from test comments to match actual function names:
- benchmark/benchmark_test.go (2 fixes)
- fileproc/filetypes_config_test.go (4 fixes)
- fileproc/filetypes_registry_test.go (6 fixes)
- fileproc/processor_test.go (6 fixes)
- fileproc/resource_monitor_types_test.go (4 fixes)
- fileproc/writer_test.go (3 fixes)

* fix: various test improvements and bug fixes

- Remove duplicate maxCacheSize check in filetypes_registry_test.go
- Shorten long comment in processor_test.go to stay under 120 chars
- Remove flaky time.Sleep in collector_test.go, use >= 0 assertion
- Close pipe reader in benchmark_test.go to fix file descriptor leak
- Use ContinueOnError in flags_test.go to match ResetFlags behavior
- Add nil check for p.ui in processor_workers.go before UpdateProgress
- Fix resource_monitor_validation_test.go by setting hardMemoryLimitBytes directly

* chore(yaml): add missing document start markers

Add --- document start to YAML files to satisfy yamllint:
- .github/workflows/codeql.yml
- .github/workflows/build-test-publish.yml
- .github/workflows/security.yml
- .github/actions/setup/action.yml

* fix: guard nil resourceMonitor and fix test deadlock

- Guard resourceMonitor before CreateFileProcessingContext call
- Add ui.UpdateProgress on emergency stop and path error returns
- Fix potential deadlock in TestProcessFile using wg.Go with defer close
2025-12-10 19:07:11 +02:00

630 lines
16 KiB
Go

package fileproc_test
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gopkg.in/yaml.v3"
"github.com/ivuorinen/gibidify/fileproc"
"github.com/ivuorinen/gibidify/shared"
)
func TestStartWriterFormats(t *testing.T) {
// Define table-driven test cases
tests := []struct {
name string
format string
expectError bool
}{
{"JSON format", "json", false},
{"YAML format", "yaml", false},
{"Markdown format", "markdown", false},
{"Invalid format", "invalid", true},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
data := runWriterTest(t, tc.format)
if tc.expectError {
verifyErrorOutput(t, data)
} else {
verifyValidOutput(t, data, tc.format)
verifyPrefixSuffix(t, data)
}
},
)
}
}
// runWriterTest executes the writer with the given format and returns the output data.
func runWriterTest(t *testing.T, format string) []byte {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "gibidify_test_output")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
defer func() {
if closeErr := outFile.Close(); closeErr != nil {
t.Errorf("close temp file: %v", closeErr)
}
if removeErr := os.Remove(outFile.Name()); removeErr != nil {
t.Errorf("remove temp file: %v", removeErr)
}
}()
// Prepare channels
writeCh := make(chan fileproc.WriteRequest, 2)
doneCh := make(chan struct{})
// Write a couple of sample requests
writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: shared.LiteralPackageMain}
writeCh <- fileproc.WriteRequest{Path: "example.py", Content: "def foo(): pass"}
close(writeCh)
// Start the writer
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
})
// Wait until writer signals completion
wg.Wait()
select {
case <-doneCh: // make sure all writes finished
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
// Read output
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Error reading output file: %v", err)
}
return data
}
// verifyErrorOutput checks that error cases produce no output.
func verifyErrorOutput(t *testing.T, data []byte) {
t.Helper()
if len(data) != 0 {
t.Errorf("Expected no output for invalid format, got:\n%s", data)
}
}
// verifyValidOutput checks format-specific output validity.
func verifyValidOutput(t *testing.T, data []byte, format string) {
t.Helper()
content := string(data)
switch format {
case "json":
var outStruct fileproc.OutputData
if err := json.Unmarshal(data, &outStruct); err != nil {
t.Errorf("JSON unmarshal failed: %v", err)
}
case "yaml":
var outStruct fileproc.OutputData
if err := yaml.Unmarshal(data, &outStruct); err != nil {
t.Errorf("YAML unmarshal failed: %v", err)
}
case "markdown":
if !strings.Contains(content, "```") {
t.Error("Expected markdown code fences not found")
}
default:
// Unknown format - basic validation that we have content
if len(content) == 0 {
t.Errorf("Unexpected format %s with empty content", format)
}
}
}
// verifyPrefixSuffix checks that output contains expected prefix and suffix.
func verifyPrefixSuffix(t *testing.T, data []byte) {
t.Helper()
content := string(data)
if !strings.Contains(content, "PREFIX") {
t.Errorf("Missing prefix in output: %s", data)
}
if !strings.Contains(content, "SUFFIX") {
t.Errorf("Missing suffix in output: %s", data)
}
}
// verifyPrefixSuffixWith checks that output contains expected custom prefix and suffix.
func verifyPrefixSuffixWith(t *testing.T, data []byte, expectedPrefix, expectedSuffix string) {
t.Helper()
content := string(data)
if !strings.Contains(content, expectedPrefix) {
t.Errorf("Missing prefix '%s' in output: %s", expectedPrefix, data)
}
if !strings.Contains(content, expectedSuffix) {
t.Errorf("Missing suffix '%s' in output: %s", expectedSuffix, data)
}
}
// TestStartWriterStreamingFormats tests streaming functionality in all writers.
func TestStartWriterStreamingFormats(t *testing.T) {
tests := []struct {
name string
format string
content string
}{
{"JSON streaming", "json", strings.Repeat("line\n", 1000)},
{"YAML streaming", "yaml", strings.Repeat("data: value\n", 1000)},
{"Markdown streaming", "markdown", strings.Repeat("# Header\nContent\n", 1000)},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
data := runStreamingWriterTest(t, tc.format, tc.content)
// Verify output is not empty
if len(data) == 0 {
t.Error("Expected streaming output but got empty result")
}
// Format-specific validation
verifyValidOutput(t, data, tc.format)
verifyPrefixSuffixWith(t, data, "STREAM_PREFIX", "STREAM_SUFFIX")
// Verify content was written
content := string(data)
if !strings.Contains(content, shared.TestFileStreamTest) {
t.Error("Expected file path in streaming output")
}
},
)
}
}
// runStreamingWriterTest executes the writer with streaming content.
func runStreamingWriterTest(t *testing.T, format, content string) []byte {
t.Helper()
// Create temp file with content for streaming
contentFile, err := os.CreateTemp(t.TempDir(), "content_*.txt")
if err != nil {
t.Fatalf("Failed to create content file: %v", err)
}
defer func() {
if err := os.Remove(contentFile.Name()); err != nil {
t.Logf("Failed to remove content file: %v", err)
}
}()
if _, err := contentFile.WriteString(content); err != nil {
t.Fatalf("Failed to write content file: %v", err)
}
if err := contentFile.Close(); err != nil {
t.Fatalf("Failed to close content file: %v", err)
}
// Create output file
outFile, err := os.CreateTemp(t.TempDir(), "gibidify_stream_test_output")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
defer func() {
if closeErr := outFile.Close(); closeErr != nil {
t.Errorf("close temp file: %v", closeErr)
}
if removeErr := os.Remove(outFile.Name()); removeErr != nil {
t.Errorf("remove temp file: %v", removeErr)
}
}()
// Prepare channels with streaming request
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
// Create reader for streaming
reader, err := os.Open(contentFile.Name())
if err != nil {
t.Fatalf("Failed to open content file for reading: %v", err)
}
defer func() {
if err := reader.Close(); err != nil {
t.Logf("Failed to close reader: %v", err)
}
}()
// Write streaming request
writeCh <- fileproc.WriteRequest{
Path: shared.TestFileStreamTest,
Content: "", // Empty for streaming
IsStream: true,
Reader: reader,
}
close(writeCh)
// Start the writer
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "STREAM_PREFIX", "STREAM_SUFFIX")
})
// Wait until writer signals completion
wg.Wait()
select {
case <-doneCh:
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
// Read output
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Error reading output file: %v", err)
}
return data
}
// setupReadOnlyFile creates a read-only file for error testing.
func setupReadOnlyFile(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outPath := filepath.Join(t.TempDir(), "readonly_out")
outFile, err := os.Create(outPath)
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
// Close writable FD and reopen as read-only so writes will fail
_ = outFile.Close()
outFile, err = os.OpenFile(outPath, os.O_RDONLY, 0)
if err != nil {
t.Fatalf("Failed to reopen as read-only: %v", err)
}
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{
Path: shared.TestFileGo,
Content: shared.LiteralPackageMain,
}
close(writeCh)
return outFile, writeCh, doneCh
}
// setupStreamingError creates a streaming request with a failing reader.
func setupStreamingError(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "yaml_stream_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
pr, pw := io.Pipe()
if err := pw.CloseWithError(errors.New("simulated stream error")); err != nil {
t.Fatalf("failed to set pipe error: %v", err)
}
writeCh <- fileproc.WriteRequest{
Path: "stream_fail.yaml",
Content: "", // Empty for streaming
IsStream: true,
Reader: pr,
}
close(writeCh)
return outFile, writeCh, doneCh
}
// setupSpecialCharacters creates requests with special characters.
func setupSpecialCharacters(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "markdown_special_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
writeCh := make(chan fileproc.WriteRequest, 2)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{
Path: "special\ncharacters.md",
Content: "Content with\x00null bytes and\ttabs",
}
writeCh <- fileproc.WriteRequest{
Path: "empty.md",
Content: "",
}
close(writeCh)
return outFile, writeCh, doneCh
}
// runErrorHandlingTest runs a single error handling test.
func runErrorHandlingTest(
t *testing.T,
outFile *os.File,
writeCh chan fileproc.WriteRequest,
doneCh chan struct{},
format string,
expectEmpty bool,
) {
t.Helper()
defer func() {
if err := os.Remove(outFile.Name()); err != nil {
t.Logf("Failed to remove temp file: %v", err)
}
}()
defer func() {
if err := outFile.Close(); err != nil {
t.Logf("Failed to close temp file: %v", err)
}
}()
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
})
wg.Wait()
// Wait for doneCh with timeout to prevent test hangs
select {
case <-doneCh:
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
// Read output file and verify based on expectation
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Failed to read output file: %v", err)
}
if expectEmpty && len(data) != 0 {
t.Errorf("expected empty output on error, got %d bytes", len(data))
}
if !expectEmpty && len(data) == 0 {
t.Error("expected non-empty output, got empty")
}
}
// TestStartWriterErrorHandling tests error scenarios in writers.
func TestStartWriterErrorHandling(t *testing.T) {
tests := []struct {
name string
format string
setupError func(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{})
expectEmptyOutput bool
}{
{
name: "JSON writer with read-only file",
format: "json",
setupError: setupReadOnlyFile,
expectEmptyOutput: true,
},
{
name: "YAML writer with streaming error",
format: "yaml",
setupError: setupStreamingError,
expectEmptyOutput: false, // Partial writes are acceptable before streaming errors
},
{
name: "Markdown writer with special characters",
format: "markdown",
setupError: setupSpecialCharacters,
expectEmptyOutput: false,
},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
outFile, writeCh, doneCh := tc.setupError(t)
runErrorHandlingTest(t, outFile, writeCh, doneCh, tc.format, tc.expectEmptyOutput)
},
)
}
}
// setupCloseTest sets up files and channels for close testing.
func setupCloseTest(t *testing.T) (*os.File, chan fileproc.WriteRequest, chan struct{}) {
t.Helper()
outFile, err := os.CreateTemp(t.TempDir(), "close_test_*")
if err != nil {
t.Fatalf(shared.TestMsgFailedToCreateFile, err)
}
writeCh := make(chan fileproc.WriteRequest, 5)
doneCh := make(chan struct{})
for i := 0; i < 5; i++ {
writeCh <- fileproc.WriteRequest{
Path: fmt.Sprintf("file%d.txt", i),
Content: fmt.Sprintf("Content %d", i),
}
}
close(writeCh)
return outFile, writeCh, doneCh
}
// runCloseTest executes writer and validates output.
func runCloseTest(
t *testing.T,
outFile *os.File,
writeCh chan fileproc.WriteRequest,
doneCh chan struct{},
format string,
) {
t.Helper()
defer func() {
if err := os.Remove(outFile.Name()); err != nil {
t.Logf("Failed to remove temp file: %v", err)
}
}()
defer func() {
if err := outFile.Close(); err != nil {
t.Logf("Failed to close temp file: %v", err)
}
}()
var wg sync.WaitGroup
wg.Go(func() {
fileproc.StartWriter(outFile, writeCh, doneCh, format, "TEST_PREFIX", "TEST_SUFFIX")
})
wg.Wait()
select {
case <-doneCh:
case <-time.After(3 * time.Second):
t.Fatal(shared.TestMsgTimeoutWriterCompletion)
}
data, err := os.ReadFile(outFile.Name())
if err != nil {
t.Fatalf("Failed to read output file: %v", err)
}
if len(data) == 0 {
t.Error("Expected non-empty output file")
}
verifyPrefixSuffixWith(t, data, "TEST_PREFIX", "TEST_SUFFIX")
}
// TestStartWriterWriterCloseErrors tests error handling during writer close operations.
func TestStartWriterWriterCloseErrors(t *testing.T) {
tests := []struct {
name string
format string
}{
{"JSON close handling", "json"},
{"YAML close handling", "yaml"},
{"Markdown close handling", "markdown"},
}
for _, tc := range tests {
t.Run(
tc.name, func(t *testing.T) {
outFile, writeCh, doneCh := setupCloseTest(t)
runCloseTest(t, outFile, writeCh, doneCh, tc.format)
},
)
}
}
// Benchmarks for writer performance
// BenchmarkStartWriter benchmarks basic writer operations across formats.
func BenchmarkStartWriter(b *testing.B) {
formats := []string{"json", "yaml", "markdown"}
for _, format := range formats {
b.Run(format, func(b *testing.B) {
for b.Loop() {
outFile, err := os.CreateTemp(b.TempDir(), "bench_output_*")
if err != nil {
b.Fatalf("Failed to create temp file: %v", err)
}
writeCh := make(chan fileproc.WriteRequest, 2)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{Path: "sample.go", Content: shared.LiteralPackageMain}
writeCh <- fileproc.WriteRequest{Path: "example.py", Content: "def foo(): pass"}
close(writeCh)
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
<-doneCh
_ = outFile.Close()
}
})
}
}
// benchStreamingIteration runs a single streaming benchmark iteration.
func benchStreamingIteration(b *testing.B, format, content string) {
b.Helper()
contentFile := createBenchContentFile(b, content)
defer func() { _ = os.Remove(contentFile) }()
reader, err := os.Open(contentFile)
if err != nil {
b.Fatalf("Failed to open content file: %v", err)
}
defer func() { _ = reader.Close() }()
outFile, err := os.CreateTemp(b.TempDir(), "bench_stream_output_*")
if err != nil {
b.Fatalf("Failed to create output file: %v", err)
}
defer func() { _ = outFile.Close() }()
writeCh := make(chan fileproc.WriteRequest, 1)
doneCh := make(chan struct{})
writeCh <- fileproc.WriteRequest{
Path: shared.TestFileStreamTest,
Content: "",
IsStream: true,
Reader: reader,
}
close(writeCh)
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
<-doneCh
}
// createBenchContentFile creates a temp file with content for benchmarks.
func createBenchContentFile(b *testing.B, content string) string {
b.Helper()
contentFile, err := os.CreateTemp(b.TempDir(), "content_*")
if err != nil {
b.Fatalf("Failed to create content file: %v", err)
}
if _, err := contentFile.WriteString(content); err != nil {
b.Fatalf("Failed to write content: %v", err)
}
if err := contentFile.Close(); err != nil {
b.Fatalf("Failed to close content file: %v", err)
}
return contentFile.Name()
}
// BenchmarkStartWriterStreaming benchmarks streaming writer operations across formats.
func BenchmarkStartWriterStreaming(b *testing.B) {
formats := []string{"json", "yaml", "markdown"}
content := strings.Repeat("line content\n", 1000)
for _, format := range formats {
b.Run(format, func(b *testing.B) {
for b.Loop() {
benchStreamingIteration(b, format, content)
}
})
}
}