Initial commit

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_size = 2
+indent_style = space
+tab_width = 2
+
+[*.md]
+trim_trailing_whitespace = false

.github/workflows/ci.yml

@@ -0,0 +1,63 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+name: Build and Publish
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  release:
+    types: [ created ]
+
+jobs:
+  build:
+    name: Build Binaries
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [ "linux", "darwin" ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.23'
+
+      - name: Build binary for ${{ matrix.goos }}
+        shell: bash
+        run: |
+          GOOS=${{ matrix.goos }} GOARCH=amd64 go build \
+            -ldflags "-X main.Version=dev-$(date -u +%Y%m%d%H%M)" \
+            -o gibidi-${{ matrix.goos }} \
+            .
+
+      - name: Upload artifact for ${{ matrix.goos }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: gibidify-${{ matrix.goos }}
+          path: gibidify-${{ matrix.goos }}
+
+  docker:
+    name: Build and Publish Docker Image
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Download Linux binary artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: gibidify-linux
+          path: ./build
+
+      - name: Build Docker image
+        shell: bash
+        run: |
+          cp ./build/gibidi-linux ./gibidi
+          chmod +x ./gibidi
+          docker build -t ghcr.io/${{ github.repository }}/gibidi:${{ github.ref_name }} .

.gitignore

@@ -0,0 +1,2 @@
+gibidify
+gibidify.txt

Dockerfile

@@ -0,0 +1,12 @@
+# Use a minimal base image
+FROM alpine:latest
+
+# Copy the gibidify binary into the container
+COPY gibidify /usr/local/bin/gibidify
+
+# Ensure the binary is executable
+RUN chmod +x /usr/local/bin/gibidify
+
+# Set the entrypoint
+ENTRYPOINT ["/usr/local/bin/gibidify"]
+

LICENSE

@@ -0,0 +1,7 @@
+MIT License Copyright (c) 2021 Ben Boyter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,79 @@
+# gibidify
+
+gibidify is a CLI application written in Go that scans a source directory
+recursively and aggregates code files into a single text file. The output
+file is designed for use with LLM models and includes a prefix,
+file sections with separators, and a suffix.
+
+## Features
+
+- Recursive scanning of a source directory.
+- File filtering based on size, glob patterns, and .gitignore rules.
+- Modular, concurrent file processing with progress bar feedback.
+- Configurable logging and configuration via Viper.
+- Cross-platform build with Docker packaging support.
+
+## Installation
+
+Clone the repository and build the application:
+
+```bash
+git clone https://github.com/ivuorinen/gibidify.git
+cd gibidify
+go build -o gibidify .
+```
+
+## Usage
+
+```bash
+./gibidify -source <source_directory> -destination <output_file> [--prefix="..."] [--suffix="..."]
+```
+
+## Docker
+
+A Docker image can be built using the provided Dockerfile:
+
+```bash
+docker build -t ghcr.io/ivuorinen/gibidify:<tag> .
+```
+
+Run the Docker container:
+
+```bash
+docker run --rm \
+  -v $(pwd):/workspace \
+  -v $HOME/.config/gibidify:/config \
+  ghcr.io/ivuorinen/gibidify:<tag> \
+  -source /workspace/your_source_directory \
+  -destination /workspace/output.txt \
+  --prefix="Your prefix text" \
+  --suffix="Your suffix text"
+```
+
+## Configuration
+
+gibidify supports a YAML configuration file. Place it at:
+
+- `$XDG_CONFIG_HOME/gibidify/config.yaml` or
+- `$HOME/.config/gibidify/config.yaml` or
+- in the folder you run the application from.
+
+Example configuration:
+
+```yaml
+fileSizeLimit: 5242880  # 5 MB
+ignoreDirectories:
+  - vendor
+  - node_modules
+  - .git
+  - dist
+  - build
+  - target
+  - bower_components
+  - cache
+  - tmp
+```
+
+## License
+
+This project is licensed under [the MIT License](LICENSE).

config/config.go

@@ -0,0 +1,53 @@
+// Package config handles application configuration using Viper.
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+)
+
+// LoadConfig reads configuration from a YAML file.
+// It looks for config in the following order:
+// 1. $XDG_CONFIG_HOME/gibidify/config.yaml
+// 2. $HOME/.config/gibidify/config.yaml
+// 3. The current directory as fallback.
+func LoadConfig() {
+	viper.SetConfigName("config")
+	viper.SetConfigType("yaml")
+
+	if xdgConfig := os.Getenv("XDG_CONFIG_HOME"); xdgConfig != "" {
+		viper.AddConfigPath(filepath.Join(xdgConfig, "gibidify"))
+	} else if home, err := os.UserHomeDir(); err == nil {
+		viper.AddConfigPath(filepath.Join(home, ".config", "gibidify"))
+	}
+	viper.AddConfigPath(".")
+
+	if err := viper.ReadInConfig(); err != nil {
+		logrus.Infof("Config file not found, using default values: %v", err)
+		setDefaultConfig()
+	} else {
+		logrus.Infof("Using config file: %s", viper.ConfigFileUsed())
+	}
+}
+
+// setDefaultConfig sets default configuration values.
+func setDefaultConfig() {
+	viper.SetDefault("fileSizeLimit", 5242880) // 5 MB
+	// Default ignored directories.
+	viper.SetDefault("ignoreDirectories", []string{
+		"vendor", "node_modules", ".git", "dist", "build", "target", "bower_components", "cache", "tmp",
+	})
+}
+
+// GetFileSizeLimit returns the file size limit from configuration.
+func GetFileSizeLimit() int64 {
+	return viper.GetInt64("fileSizeLimit")
+}
+
+// GetIgnoredDirectories returns the list of directories to ignore.
+func GetIgnoredDirectories() []string {
+	return viper.GetStringSlice("ignoreDirectories")
+}

fileproc/collector.go

@@ -0,0 +1,9 @@
+// Package fileproc provides functions for collecting and processing files.
+package fileproc
+
+// CollectFiles scans the given root directory using the default walker (ProdWalker)
+// and returns a slice of file paths.
+func CollectFiles(root string) ([]string, error) {
+	var w Walker = ProdWalker{}
+	return w.Walk(root)
+}

fileproc/collector_test.go

@@ -0,0 +1,47 @@
+package fileproc
+
+import (
+	"os"
+	"testing"
+)
+
+func TestCollectFilesWithFakeWalker(t *testing.T) {
+	// Instead of using the production walker, use FakeWalker.
+	expectedFiles := []string{
+		"/path/to/file1.txt",
+		"/path/to/file2.go",
+	}
+	fake := FakeWalker{
+		Files: expectedFiles,
+		Err:   nil,
+	}
+
+	// Use fake.Walk directly.
+	files, err := fake.Walk("dummyRoot")
+	if err != nil {
+		t.Fatalf("Expected no error, got %v", err)
+	}
+
+	if len(files) != len(expectedFiles) {
+		t.Fatalf("Expected %d files, got %d", len(expectedFiles), len(files))
+	}
+
+	for i, f := range files {
+		if f != expectedFiles[i] {
+			t.Errorf("Expected file %s, got %s", expectedFiles[i], f)
+		}
+	}
+}
+
+func TestCollectFilesError(t *testing.T) {
+	// Fake walker returns an error.
+	fake := FakeWalker{
+		Files: nil,
+		Err:   os.ErrNotExist,
+	}
+
+	_, err := fake.Walk("dummyRoot")
+	if err == nil {
+		t.Fatal("Expected an error, got nil")
+	}
+}

fileproc/fake_walker.go

@@ -0,0 +1,16 @@
+// Package fileproc provides functions for file processing.
+package fileproc
+
+// FakeWalker implements Walker for testing purposes.
+type FakeWalker struct {
+	Files []string
+	Err   error
+}
+
+// Walk returns predetermined file paths or an error, depending on FakeWalker's configuration.
+func (fw FakeWalker) Walk(root string) ([]string, error) {
+	if fw.Err != nil {
+		return nil, fw.Err
+	}
+	return fw.Files, nil
+}

fileproc/processor.go

@@ -0,0 +1,27 @@
+// Package fileproc provides functions for processing files.
+package fileproc
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/sirupsen/logrus"
+)
+
+// WriteRequest represents the content to be written.
+type WriteRequest struct {
+	Content string
+}
+
+// ProcessFile reads the file at filePath and sends a formatted output to outCh.
+// The optional wg parameter is used when the caller wants to wait on file-level processing.
+func ProcessFile(filePath string, outCh chan<- WriteRequest, wg *interface{}) {
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		logrus.Errorf("Failed to read file %s: %v", filePath, err)
+		return
+	}
+	// Format: separator, file path, then content.
+	formatted := fmt.Sprintf("\n---\n%s\n%s\n", filePath, string(content))
+	outCh <- WriteRequest{Content: formatted}
+}

fileproc/processor_test.go

@@ -0,0 +1,45 @@
+package fileproc
+
+import (
+	"os"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestProcessFile(t *testing.T) {
+	// Create a temporary file with known content.
+	tmpFile, err := os.CreateTemp("", "testfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	content := "Test content"
+	if _, err := tmpFile.WriteString(content); err != nil {
+		t.Fatal(err)
+	}
+	tmpFile.Close()
+
+	ch := make(chan WriteRequest, 1)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ProcessFile(tmpFile.Name(), ch, nil)
+	}()
+	wg.Wait()
+	close(ch)
+
+	var result string
+	for req := range ch {
+		result = req.Content
+	}
+
+	if !strings.Contains(result, tmpFile.Name()) {
+		t.Errorf("Output does not contain file path: %s", tmpFile.Name())
+	}
+	if !strings.Contains(result, content) {
+		t.Errorf("Output does not contain file content: %s", content)
+	}
+}

fileproc/walker.go

@@ -0,0 +1,40 @@
+// Package fileproc provides functions for file processing.
+package fileproc
+
+import (
+	"github.com/boyter/gocodewalker"
+	"github.com/sirupsen/logrus"
+)
+
+// Walker defines an interface for scanning directories.
+type Walker interface {
+	Walk(root string) ([]string, error)
+}
+
+// ProdWalker implements Walker using gocodewalker.
+type ProdWalker struct{}
+
+// Walk scans the given root directory using gocodewalker and returns a slice of file paths.
+func (pw ProdWalker) Walk(root string) ([]string, error) {
+	fileListQueue := make(chan *gocodewalker.File, 100)
+	fileWalker := gocodewalker.NewFileWalker(root, fileListQueue)
+
+	errorHandler := func(err error) bool {
+		logrus.Errorf("error walking directory: %s", err.Error())
+		return true
+	}
+	fileWalker.SetErrorHandler(errorHandler)
+	go func() {
+		err := fileWalker.Start()
+		if err != nil {
+			logrus.Errorf("error walking directory: %s", err.Error())
+		}
+	}()
+
+	var files []string
+	for f := range fileListQueue {
+		files = append(files, f.Location)
+	}
+
+	return files, nil
+}

fileproc/writer.go

@@ -0,0 +1,21 @@
+// Package fileproc provides functions for writing file contents concurrently.
+package fileproc
+
+import (
+	"io"
+	"os"
+
+	"github.com/sirupsen/logrus"
+)
+
+// StartWriter listens on the write channel and writes content to outFile.
+// When finished, it signals on the done channel.
+func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}) {
+	writer := io.Writer(outFile)
+	for req := range writeCh {
+		if _, err := writer.Write([]byte(req.Content)); err != nil {
+			logrus.Errorf("Error writing to file: %v", err)
+		}
+	}
+	done <- struct{}{}
+}

fileproc/writer_test.go

@@ -0,0 +1,31 @@
+package fileproc
+
+import (
+	"bytes"
+	"sync"
+	"testing"
+)
+
+func TestStartWriter(t *testing.T) {
+	var buf bytes.Buffer
+
+	writeCh := make(chan WriteRequest)
+	done := make(chan struct{})
+
+	go StartWriter(&buf, writeCh, done)
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		writeCh <- WriteRequest{Content: "Hello"}
+		writeCh <- WriteRequest{Content: " World"}
+	}()
+	wg.Wait()
+	close(writeCh)
+	<-done
+
+	if buf.String() != "Hello World" {
+		t.Errorf("Expected 'Hello World', got '%s'", buf.String())
+	}
+}

go.mod

@@ -0,0 +1,37 @@
+module github.com/ivuorinen/gibidify
+
+go 1.23
+
+require (
+	github.com/boyter/gocodewalker v1.4.0
+	github.com/schollz/progressbar/v3 v3.18.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/viper v1.19.0
+)
+
+require (
+	github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,93 @@
+github.com/boyter/gocodewalker v1.4.0 h1:fVmFeQxKpj5tlpjPcyTtJ96btgaHYd9yn6m+T/66et4=
+github.com/boyter/gocodewalker v1.4.0/go.mod h1:hXG8xzR1uURS+99P5/3xh3uWHjaV2XfoMMmvPyhrCDg=
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
+github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
+github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
+github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
+github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

integration_test.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestIntegrationFullCLI simulates a full run of the CLI application using adaptive concurrency.
+func TestIntegrationFullCLI(t *testing.T) {
+	// Create a temporary source directory and populate it with test files.
+	srcDir, err := ioutil.TempDir("", "gibidi_src")
+	if err != nil {
+		t.Fatalf("Failed to create temp source directory: %v", err)
+	}
+	defer os.RemoveAll(srcDir)
+
+	// Create two test files.
+	file1 := filepath.Join(srcDir, "file1.txt")
+	if err := ioutil.WriteFile(file1, []byte("Hello World"), 0644); err != nil {
+		t.Fatalf("Failed to write file1: %v", err)
+	}
+	file2 := filepath.Join(srcDir, "file2.go")
+	if err := ioutil.WriteFile(file2, []byte("package main\nfunc main() {}"), 0644); err != nil {
+		t.Fatalf("Failed to write file2: %v", err)
+	}
+
+	// Create a temporary output file.
+	outFile, err := ioutil.TempFile("", "gibidi_output.txt")
+	if err != nil {
+		t.Fatalf("Failed to create temp output file: %v", err)
+	}
+	outFilePath := outFile.Name()
+	outFile.Close()
+	defer os.Remove(outFilePath)
+
+	// Set up CLI arguments.
+	os.Args = []string{
+		"gibidify",
+		"-source", srcDir,
+		"-destination", outFilePath,
+		"-prefix", "PREFIX",
+		"-suffix", "SUFFIX",
+		"-concurrency", "2", // For testing, set concurrency to 2.
+	}
+
+	// Run the application with a background context.
+	ctx := context.Background()
+	if err := Run(ctx); err != nil {
+		t.Fatalf("Run failed: %v", err)
+	}
+
+	// Verify the output file contains the expected prefix, file contents, and suffix.
+	data, err := ioutil.ReadFile(outFilePath)
+	if err != nil {
+		t.Fatalf("Failed to read output file: %v", err)
+	}
+	output := string(data)
+	if !strings.Contains(output, "PREFIX") {
+		t.Error("Output missing prefix")
+	}
+	if !strings.Contains(output, "Hello World") {
+		t.Error("Output missing content from file1.txt")
+	}
+	if !strings.Contains(output, "SUFFIX") {
+		t.Error("Output missing suffix")
+	}
+}
+
+// TestIntegrationCancellation verifies that the application correctly cancels processing when the context times out.
+func TestIntegrationCancellation(t *testing.T) {
+	// Create a temporary source directory with many files to simulate a long-running process.
+	srcDir, err := ioutil.TempDir("", "gibidi_src_long")
+	if err != nil {
+		t.Fatalf("Failed to create temp source directory: %v", err)
+	}
+	defer os.RemoveAll(srcDir)
+
+	// Create a large number of small files.
+	for i := 0; i < 1000; i++ {
+		filePath := filepath.Join(srcDir, fmt.Sprintf("file%d.txt", i))
+		if err := ioutil.WriteFile(filePath, []byte("Content"), 0644); err != nil {
+			t.Fatalf("Failed to write %s: %v", filePath, err)
+		}
+	}
+
+	// Create a temporary output file.
+	outFile, err := ioutil.TempFile("", "gibidi_output.txt")
+	if err != nil {
+		t.Fatalf("Failed to create temp output file: %v", err)
+	}
+	outFilePath := outFile.Name()
+	outFile.Close()
+	defer os.Remove(outFilePath)
+
+	// Set up CLI arguments.
+	os.Args = []string{
+		"gibidify",
+		"-source", srcDir,
+		"-destination", outFilePath,
+		"-prefix", "PREFIX",
+		"-suffix", "SUFFIX",
+		"-concurrency", "2",
+	}
+
+	// Create a context with a very short timeout to force cancellation.
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+	defer cancel()
+
+	// Run the application; we expect an error due to cancellation.
+	err = Run(ctx)
+	if err == nil {
+		t.Error("Expected Run to fail due to cancellation, but it succeeded")
+	}
+}

main.go

@@ -0,0 +1,159 @@
+// Package main for the gibidify CLI application.
+// Repository: github.com/ivuorinen/gibidify
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+
+	"github.com/ivuorinen/gibidify/config"
+	"github.com/ivuorinen/gibidify/fileproc"
+	"github.com/schollz/progressbar/v3"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	sourceDir   string
+	destination string
+	prefix      string
+	suffix      string
+	concurrency int
+)
+
+func init() {
+	flag.StringVar(&sourceDir, "source", "", "Source directory to scan recursively")
+	flag.StringVar(&destination, "destination", "", "Output file to write aggregated code")
+	flag.StringVar(&prefix, "prefix", "", "Text to add at the beginning of the output file")
+	flag.StringVar(&suffix, "suffix", "", "Text to add at the end of the output file")
+	flag.IntVar(&concurrency, "concurrency", runtime.NumCPU(), "Number of concurrent workers (default: number of CPU cores)")
+}
+
+// Run executes the main logic of the CLI application using the provided context.
+func Run(ctx context.Context) error {
+	flag.Parse()
+
+	if sourceDir == "" || destination == "" {
+		return fmt.Errorf(
+			"usage: gibidify " +
+				"-source <source_directory> " +
+				"-destination <output_file> " +
+				"[--prefix=\"...\"] " +
+				"[--suffix=\"...\"] " +
+				"[-concurrency=<num>]",
+		)
+	}
+
+	// Load configuration using Viper.
+	config.LoadConfig()
+
+	logrus.Infof(
+		"Starting gibidify. Source: %s, Destination: %s, Workers: %d",
+		sourceDir,
+		destination,
+		concurrency,
+	)
+
+	// 1. Collect files using the file walker (ProdWalker).
+	files, err := fileproc.CollectFiles(sourceDir)
+	if err != nil {
+		return fmt.Errorf("error collecting files: %w", err)
+	}
+	logrus.Infof("Found %d files to process", len(files))
+
+	// 2. Open the destination file and write the header.
+	outFile, err := os.Create(destination)
+	if err != nil {
+		return fmt.Errorf("failed to create output file %s: %w", destination, err)
+	}
+	defer func(outFile *os.File) {
+		err := outFile.Close()
+		if err != nil {
+			logrus.Errorf("failed to close output file %s: %v", destination, err)
+		}
+	}(outFile)
+
+	header := prefix + "\n" +
+		"The following text is a Git repository with code. " +
+		"The structure of the text are sections that begin with ----, " +
+		"followed by a single line containing the file path and file name, " +
+		"followed by a variable amount of lines containing the file contents. " +
+		"The text representing the Git repository ends when the symbols --END-- are encountered.\n"
+
+	if _, err := outFile.WriteString(header); err != nil {
+		return fmt.Errorf("failed to write header: %w", err)
+	}
+
+	// 3. Set up channels and a worker pool for processing files.
+	fileCh := make(chan string)
+	writeCh := make(chan fileproc.WriteRequest)
+	var wg sync.WaitGroup
+
+	// Start the writer goroutine.
+	writerDone := make(chan struct{})
+	go fileproc.StartWriter(outFile, writeCh, writerDone)
+
+	// Start worker goroutines.
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case fp, ok := <-fileCh:
+					if !ok {
+						return
+					}
+					// Process the file.
+					fileproc.ProcessFile(fp, writeCh, nil)
+				case <-ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	// Feed file paths to the worker pool with progress bar feedback.
+	bar := progressbar.Default(int64(len(files)))
+loop:
+	for _, fp := range files {
+		select {
+		case fileCh <- fp:
+			_ = bar.Add(1)
+		case <-ctx.Done():
+			close(fileCh)
+			break loop
+		}
+	}
+	close(fileCh)
+
+	// Wait for all workers to finish.
+	wg.Wait()
+	close(writeCh)
+	<-writerDone
+
+	// Check for context cancellation.
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	// 4. Write footer.
+	footer := "--END--\n" + suffix
+	if _, err := outFile.WriteString(footer); err != nil {
+		return fmt.Errorf("failed to write footer: %w", err)
+	}
+
+	logrus.Infof("Processing completed. Output saved to %s", destination)
+	return nil
+}
+
+func main() {
+	// In production, use a background context.
+	if err := Run(context.Background()); err != nil {
+		fmt.Println("Error:", err)
+		os.Exit(1)
+	}
+}