feat: many features, check TODO.md

fileproc/processor.go

@@ -3,34 +3,157 @@ package fileproc
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
+	"strings"
 
-	"github.com/sirupsen/logrus"
+	"github.com/ivuorinen/gibidify/config"
+	"github.com/ivuorinen/gibidify/utils"
+)
+
+const (
+	// StreamChunkSize is the size of chunks when streaming large files (64KB).
+	StreamChunkSize = 65536
+	// StreamThreshold is the file size above which we use streaming (1MB).
+	StreamThreshold = 1048576
+	// MaxMemoryBuffer is the maximum memory to use for buffering content (10MB).
+	MaxMemoryBuffer = 10485760
 )
 
 // WriteRequest represents the content to be written.
 type WriteRequest struct {
-	Path    string
-	Content string
+	Path     string
+	Content  string
+	IsStream bool
+	Reader   io.Reader
+}
+
+// FileProcessor handles file processing operations.
+type FileProcessor struct {
+	rootPath  string
+	sizeLimit int64
+}
+
+// NewFileProcessor creates a new file processor.
+func NewFileProcessor(rootPath string) *FileProcessor {
+	return &FileProcessor{
+		rootPath:  rootPath,
+		sizeLimit: config.GetFileSizeLimit(),
+	}
 }
 
 // ProcessFile reads the file at filePath and sends a formatted output to outCh.
+// It automatically chooses between loading the entire file or streaming based on file size.
 func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) {
-	content, err := os.ReadFile(filePath)
+	processor := NewFileProcessor(rootPath)
+	processor.Process(filePath, outCh)
+}
+
+// Process handles file processing with the configured settings.
+func (p *FileProcessor) Process(filePath string, outCh chan<- WriteRequest) {
+	// Validate file
+	fileInfo, err := p.validateFile(filePath)
 	if err != nil {
-		logrus.Errorf("Failed to read file %s: %v", filePath, err)
+		return // Error already logged
+	}
+
+	// Get relative path
+	relPath := p.getRelativePath(filePath)
+
+	// Choose processing strategy based on file size
+	if fileInfo.Size() <= StreamThreshold {
+		p.processInMemory(filePath, relPath, outCh)
+	} else {
+		p.processStreaming(filePath, relPath, outCh)
+	}
+}
+
+// validateFile checks if the file can be processed.
+func (p *FileProcessor) validateFile(filePath string) (os.FileInfo, error) {
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		structErr := utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to stat file").WithFilePath(filePath)
+		utils.LogErrorf(structErr, "Failed to stat file %s", filePath)
+		return nil, err
+	}
+
+	// Check size limit
+	if fileInfo.Size() > p.sizeLimit {
+		utils.LogErrorf(
+			utils.NewStructuredError(
+				utils.ErrorTypeValidation,
+				utils.CodeValidationSize,
+				fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", fileInfo.Size(), p.sizeLimit),
+			).WithFilePath(filePath).WithContext("file_size", fileInfo.Size()).WithContext("size_limit", p.sizeLimit),
+			"Skipping large file %s", filePath,
+		)
+		return nil, fmt.Errorf("file too large")
+	}
+
+	return fileInfo, nil
+}
+
+// getRelativePath computes the path relative to rootPath.
+func (p *FileProcessor) getRelativePath(filePath string) string {
+	relPath, err := filepath.Rel(p.rootPath, filePath)
+	if err != nil {
+		return filePath // Fallback
+	}
+	return relPath
+}
+
+// processInMemory loads the entire file into memory (for small files).
+func (p *FileProcessor) processInMemory(filePath, relPath string, outCh chan<- WriteRequest) {
+	content, err := os.ReadFile(filePath) // #nosec G304 - filePath is validated by walker
+	if err != nil {
+		structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to read file").WithFilePath(filePath)
+		utils.LogErrorf(structErr, "Failed to read file %s", filePath)
 		return
 	}
 
-	// Compute path relative to rootPath, so /a/b/c/d.c becomes c/d.c
-	relPath, err := filepath.Rel(rootPath, filePath)
-	if err != nil {
-		// Fallback if something unexpected happens
-		relPath = filePath
+	outCh <- WriteRequest{
+		Path:     relPath,
+		Content:  p.formatContent(relPath, string(content)),
+		IsStream: false,
+	}
+}
+
+// processStreaming creates a streaming reader for large files.
+func (p *FileProcessor) processStreaming(filePath, relPath string, outCh chan<- WriteRequest) {
+	reader := p.createStreamReader(filePath, relPath)
+	if reader == nil {
+		return // Error already logged
 	}
 
-	// Format: separator, then relative path, then content
-	formatted := fmt.Sprintf("\n---\n%s\n%s\n", relPath, string(content))
-	outCh <- WriteRequest{Path: relPath, Content: formatted}
+	outCh <- WriteRequest{
+		Path:     relPath,
+		Content:  "", // Empty since content is in Reader
+		IsStream: true,
+		Reader:   reader,
+	}
+}
+
+// createStreamReader creates a reader that combines header and file content.
+func (p *FileProcessor) createStreamReader(filePath, relPath string) io.Reader {
+	file, err := os.Open(filePath) // #nosec G304 - filePath is validated by walker
+	if err != nil {
+		structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to open file for streaming").WithFilePath(filePath)
+		utils.LogErrorf(structErr, "Failed to open file for streaming %s", filePath)
+		return nil
+	}
+	// Note: file will be closed by the writer
+
+	header := p.formatHeader(relPath)
+	return io.MultiReader(header, file)
+}
+
+// formatContent formats the file content with header.
+func (p *FileProcessor) formatContent(relPath, content string) string {
+	return fmt.Sprintf("\n---\n%s\n%s\n", relPath, content)
+}
+
+// formatHeader creates a reader for the file header.
+func (p *FileProcessor) formatHeader(relPath string) io.Reader {
+	return strings.NewReader(fmt.Sprintf("\n---\n%s\n", relPath))
 }