diff --git a/.editorconfig b/.editorconfig index 6ad84df..86d1176 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,7 +5,7 @@ end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true indent_size = 2 -indent_style = space +indent_style = tab tab_width = 2 [*.md] diff --git a/.gitignore b/.gitignore index 3c1e831..03d459f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,9 @@ +.DS_Store +.idea gibidify +gibidify.json gibidify.txt +gibidify.yaml +output.json +output.txt +output.yaml diff --git a/Dockerfile b/Dockerfile index 9c00844..e272916 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,3 @@ RUN chmod +x /usr/local/bin/gibidify # Set the entrypoint ENTRYPOINT ["/usr/local/bin/gibidify"] - diff --git a/LICENSE b/LICENSE index 6be46d4..bea2d15 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -MIT License Copyright (c) 2021 Ben Boyter +MIT License Copyright (c) 2025 Ismo Vuorinen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/fileproc/processor.go b/fileproc/processor.go index f21887c..97e0761 100644 --- a/fileproc/processor.go +++ b/fileproc/processor.go @@ -3,25 +3,34 @@ package fileproc import ( "fmt" - "io/ioutil" + "os" + "path/filepath" "github.com/sirupsen/logrus" ) // WriteRequest represents the content to be written. type WriteRequest struct { + Path string Content string } // ProcessFile reads the file at filePath and sends a formatted output to outCh. -// The optional wg parameter is used when the caller wants to wait on file-level processing. -func ProcessFile(filePath string, outCh chan<- WriteRequest, wg *interface{}) { - content, err := ioutil.ReadFile(filePath) +func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) { + content, err := os.ReadFile(filePath) if err != nil { logrus.Errorf("Failed to read file %s: %v", filePath, err) return } - // Format: separator, file path, then content. - formatted := fmt.Sprintf("\n---\n%s\n%s\n", filePath, string(content)) - outCh <- WriteRequest{Content: formatted} + + // Compute path relative to rootPath, so /a/b/c/d.c becomes c/d.c + relPath, err := filepath.Rel(rootPath, filePath) + if err != nil { + // Fallback if something unexpected happens + relPath = filePath + } + + // Format: separator, then relative path, then content + formatted := fmt.Sprintf("\n---\n%s\n%s\n", relPath, string(content)) + outCh <- WriteRequest{Path: relPath, Content: formatted} } diff --git a/fileproc/processor_test.go b/fileproc/processor_test.go index 4dc3152..057e424 100644 --- a/fileproc/processor_test.go +++ b/fileproc/processor_test.go @@ -13,20 +13,29 @@ func TestProcessFile(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(tmpFile.Name()) + defer func(name string) { + err := os.Remove(name) + if err != nil { + t.Fatal(err) + } + }(tmpFile.Name()) content := "Test content" if _, err := tmpFile.WriteString(content); err != nil { t.Fatal(err) } - tmpFile.Close() + errTmpFile := tmpFile.Close() + if errTmpFile != nil { + t.Fatal(errTmpFile) + return + } ch := make(chan WriteRequest, 1) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() - ProcessFile(tmpFile.Name(), ch, nil) + ProcessFile(tmpFile.Name(), ch, "") }() wg.Wait() close(ch) diff --git a/fileproc/walker.go b/fileproc/walker.go index e1c42da..f4bb776 100644 --- a/fileproc/walker.go +++ b/fileproc/walker.go @@ -2,8 +2,12 @@ package fileproc import ( - "github.com/boyter/gocodewalker" - "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" + + "github.com/ivuorinen/gibidify/config" + ignore "github.com/sabhiram/go-gitignore" ) // Walker defines an interface for scanning directories. @@ -11,30 +15,148 @@ type Walker interface { Walk(root string) ([]string, error) } -// ProdWalker implements Walker using gocodewalker. +// ProdWalker implements Walker using a custom directory walker that +// respects .gitignore and .ignore files, configuration-defined ignore directories, +// and ignores binary and image files by default. type ProdWalker struct{} -// Walk scans the given root directory using gocodewalker and returns a slice of file paths. -func (pw ProdWalker) Walk(root string) ([]string, error) { - fileListQueue := make(chan *gocodewalker.File, 100) - fileWalker := gocodewalker.NewFileWalker(root, fileListQueue) +// ignoreRule holds an ignore matcher along with the base directory where it was loaded. +type ignoreRule struct { + base string + gi *ignore.GitIgnore +} - errorHandler := func(err error) bool { - logrus.Errorf("error walking directory: %s", err.Error()) +// Walk scans the given root directory recursively and returns a slice of file paths +// that are not ignored based on .gitignore/.ignore files, the configuration, or the default binary/image filter. +func (pw ProdWalker) Walk(root string) ([]string, error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, err + } + return walkDir(absRoot, absRoot, []ignoreRule{}) +} + +// walkDir recursively walks the directory tree starting at currentDir. +// It loads any .gitignore and .ignore files found in each directory and +// appends the corresponding rules to the inherited list. Each file/directory is +// then checked against the accumulated ignore rules, the configuration's list of ignored directories, +// and a default filter that ignores binary and image files. +func walkDir(root string, currentDir string, parentRules []ignoreRule) ([]string, error) { + var results []string + + entries, err := os.ReadDir(currentDir) + if err != nil { + return nil, err + } + + // Start with the parent's ignore rules. + rules := make([]ignoreRule, len(parentRules)) + copy(rules, parentRules) + + // Check for .gitignore and .ignore files in the current directory. + for _, fileName := range []string{".gitignore", ".ignore"} { + ignorePath := filepath.Join(currentDir, fileName) + if info, err := os.Stat(ignorePath); err == nil && !info.IsDir() { + gi, err := ignore.CompileIgnoreFile(ignorePath) + if err == nil { + rules = append(rules, ignoreRule{ + base: currentDir, + gi: gi, + }) + } + } + } + + // Get the list of directories to ignore from configuration. + ignoredDirs := config.GetIgnoredDirectories() + sizeLimit := config.GetFileSizeLimit() // e.g., 5242880 for 5 MB + + for _, entry := range entries { + fullPath := filepath.Join(currentDir, entry.Name()) + + // For directories, check if its name is in the config ignore list. + if entry.IsDir() { + for _, d := range ignoredDirs { + if entry.Name() == d { + // Skip this directory entirely. + goto SkipEntry + } + } + } else { + // Check if file exceeds the configured size limit. + info, err := entry.Info() + if err == nil && info.Size() > sizeLimit { + goto SkipEntry + } + + // For files, apply the default filter to ignore binary and image files. + if isBinaryOrImage(fullPath) { + goto SkipEntry + } + } + + // Check accumulated ignore rules. + for _, rule := range rules { + // Compute the path relative to the base where the ignore rule was defined. + rel, err := filepath.Rel(rule.base, fullPath) + if err != nil { + continue + } + // If the rule matches, skip this entry. + if rule.gi.MatchesPath(rel) { + goto SkipEntry + } + } + + // If not ignored, then process the entry. + if entry.IsDir() { + subFiles, err := walkDir(root, fullPath, rules) + if err != nil { + return nil, err + } + results = append(results, subFiles...) + } else { + results = append(results, fullPath) + } + SkipEntry: + continue + } + + return results, nil +} + +// isBinaryOrImage checks if a file should be considered binary or an image based on its extension. +// The check is case-insensitive. +func isBinaryOrImage(filePath string) bool { + ext := strings.ToLower(filepath.Ext(filePath)) + // Common image file extensions. + imageExtensions := map[string]bool{ + ".png": true, + ".jpg": true, + ".jpeg": true, + ".gif": true, + ".bmp": true, + ".tiff": true, + ".ico": true, + ".svg": true, + ".webp": true, + } + // Common binary file extensions. + binaryExtensions := map[string]bool{ + ".exe": true, + ".dll": true, + ".so": true, + ".bin": true, + ".dat": true, + ".zip": true, + ".tar": true, + ".gz": true, + ".7z": true, + ".rar": true, + ".DS_Store": true, + } + if imageExtensions[ext] || binaryExtensions[ext] { return true } - fileWalker.SetErrorHandler(errorHandler) - go func() { - err := fileWalker.Start() - if err != nil { - logrus.Errorf("error walking directory: %s", err.Error()) - } - }() - - var files []string - for f := range fileListQueue { - files = append(files, f.Location) - } - - return files, nil + return false } diff --git a/fileproc/writer.go b/fileproc/writer.go index 5076812..b9d929c 100644 --- a/fileproc/writer.go +++ b/fileproc/writer.go @@ -1,21 +1,94 @@ -// Package fileproc provides functions for writing file contents concurrently. package fileproc import ( - "io" + "encoding/json" + "fmt" "os" "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" ) -// StartWriter listens on the write channel and writes content to outFile. -// When finished, it signals on the done channel. -func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}) { - writer := io.Writer(outFile) - for req := range writeCh { - if _, err := writer.Write([]byte(req.Content)); err != nil { - logrus.Errorf("Error writing to file: %v", err) - } - } - done <- struct{}{} +// FileData represents a single file's path and content. +type FileData struct { + Path string `json:"path" yaml:"path"` + Content string `json:"content" yaml:"content"` +} + +// OutputData represents the full output structure. +type OutputData struct { + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + Files []FileData `json:"files" yaml:"files"` + Suffix string `json:"suffix,omitempty" yaml:"suffix,omitempty"` +} + +// StartWriter writes the output in the specified format. +func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, format string, prefix, suffix string) { + var files []FileData + + // Read from channel until closed + for req := range writeCh { + files = append(files, FileData{Path: req.Path, Content: req.Content}) + } + + // Create output struct + output := OutputData{Prefix: prefix, Files: files, Suffix: suffix} + + // Serialize based on format + var outputData []byte + var err error + + switch format { + case "json": + outputData, err = json.MarshalIndent(output, "", " ") + case "yaml": + outputData, err = yaml.Marshal(output) + case "markdown": + outputData = []byte(formatMarkdown(output)) + default: + err = fmt.Errorf("unsupported format: %s", format) + } + + if err != nil { + logrus.Errorf("Error encoding output: %v", err) + close(done) + return + } + + // Write to file + if _, err := outFile.Write(outputData); err != nil { + logrus.Errorf("Error writing to file: %v", err) + } + + close(done) +} + +func formatMarkdown(output OutputData) string { + markdown := "# " + output.Prefix + "\n\n" + + for _, file := range output.Files { + markdown += fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", file.Path, detectLanguage(file.Path), file.Content) + } + + markdown += "# " + output.Suffix + return markdown +} + +// detectLanguage tries to infer code block language from file extension. +func detectLanguage(filename string) string { + if len(filename) < 3 { + return "" + } + switch { + case len(filename) >= 3 && filename[len(filename)-3:] == ".go": + return "go" + case len(filename) >= 3 && filename[len(filename)-3:] == ".py": + return "python" + case len(filename) >= 2 && filename[len(filename)-2:] == ".c": + return "c" + case len(filename) >= 3 && filename[len(filename)-3:] == ".js": + return "javascript" + default: + return "" + } } diff --git a/fileproc/writer_test.go b/fileproc/writer_test.go index a672a8a..ab845af 100644 --- a/fileproc/writer_test.go +++ b/fileproc/writer_test.go @@ -1,31 +1,45 @@ package fileproc import ( - "bytes" - "sync" + "encoding/json" + "os" "testing" ) -func TestStartWriter(t *testing.T) { - var buf bytes.Buffer +func TestStartWriter_JSONOutput(t *testing.T) { + outFile, err := os.CreateTemp("", "output.json") + if err != nil { + t.Fatal(err) + } + defer func(name string) { + err := os.Remove(name) + if err != nil { + t.Fatal(err) + } + }(outFile.Name()) writeCh := make(chan WriteRequest) done := make(chan struct{}) - go StartWriter(&buf, writeCh, done) + go StartWriter(outFile, writeCh, done, "json", "Prefix", "Suffix") + + writeCh <- WriteRequest{Path: "file1.go", Content: "package main"} + writeCh <- WriteRequest{Path: "file2.py", Content: "def hello(): print('Hello')"} - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - writeCh <- WriteRequest{Content: "Hello"} - writeCh <- WriteRequest{Content: " World"} - }() - wg.Wait() close(writeCh) <-done - if buf.String() != "Hello World" { - t.Errorf("Expected 'Hello World', got '%s'", buf.String()) + data, err := os.ReadFile(outFile.Name()) + if err != nil { + t.Fatal(err) + } + + var output OutputData + if err := json.Unmarshal(data, &output); err != nil { + t.Fatalf("JSON output is invalid: %v", err) + } + + if len(output.Files) != 2 { + t.Errorf("Expected 2 files, got %d", len(output.Files)) } } diff --git a/go.mod b/go.mod index f44623c..378f50b 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,18 @@ module github.com/ivuorinen/gibidify go 1.23 require ( - github.com/boyter/gocodewalker v1.4.0 - github.com/schollz/progressbar/v3 v3.18.0 + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -27,11 +24,10 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.29.0 // indirect - golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index b73f39c..219ba2f 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,3 @@ -github.com/boyter/gocodewalker v1.4.0 h1:fVmFeQxKpj5tlpjPcyTtJ96btgaHYd9yn6m+T/66et4= -github.com/boyter/gocodewalker v1.4.0/go.mod h1:hXG8xzR1uURS+99P5/3xh3uWHjaV2XfoMMmvPyhrCDg= -github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= -github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -22,10 +16,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= @@ -33,16 +23,14 @@ github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= -github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= -github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -60,6 +48,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -72,15 +61,15 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/integration_test.go b/integration_test.go index de13ae9..e4f9e01 100644 --- a/integration_test.go +++ b/integration_test.go @@ -14,7 +14,7 @@ import ( // TestIntegrationFullCLI simulates a full run of the CLI application using adaptive concurrency. func TestIntegrationFullCLI(t *testing.T) { // Create a temporary source directory and populate it with test files. - srcDir, err := ioutil.TempDir("", "gibidi_src") + srcDir, err := ioutil.TempDir("", "gibidify_src") if err != nil { t.Fatalf("Failed to create temp source directory: %v", err) } @@ -31,7 +31,7 @@ func TestIntegrationFullCLI(t *testing.T) { } // Create a temporary output file. - outFile, err := ioutil.TempFile("", "gibidi_output.txt") + outFile, err := ioutil.TempFile("", "gibidify_output.txt") if err != nil { t.Fatalf("Failed to create temp output file: %v", err) } @@ -75,7 +75,7 @@ func TestIntegrationFullCLI(t *testing.T) { // TestIntegrationCancellation verifies that the application correctly cancels processing when the context times out. func TestIntegrationCancellation(t *testing.T) { // Create a temporary source directory with many files to simulate a long-running process. - srcDir, err := ioutil.TempDir("", "gibidi_src_long") + srcDir, err := ioutil.TempDir("", "gibidify_src_long") if err != nil { t.Fatalf("Failed to create temp source directory: %v", err) } @@ -90,7 +90,7 @@ func TestIntegrationCancellation(t *testing.T) { } // Create a temporary output file. - outFile, err := ioutil.TempFile("", "gibidi_output.txt") + outFile, err := ioutil.TempFile("", "gibidify_output.txt") if err != nil { t.Fatalf("Failed to create temp output file: %v", err) } diff --git a/main.go b/main.go index b828982..5ef473e 100644 --- a/main.go +++ b/main.go @@ -7,12 +7,12 @@ import ( "flag" "fmt" "os" + "path/filepath" "runtime" "sync" "github.com/ivuorinen/gibidify/config" "github.com/ivuorinen/gibidify/fileproc" - "github.com/schollz/progressbar/v3" "github.com/sirupsen/logrus" ) @@ -22,6 +22,7 @@ var ( prefix string suffix string concurrency int + format string ) func init() { @@ -29,6 +30,7 @@ func init() { flag.StringVar(&destination, "destination", "", "Output file to write aggregated code") flag.StringVar(&prefix, "prefix", "", "Text to add at the beginning of the output file") flag.StringVar(&suffix, "suffix", "", "Text to add at the end of the output file") + flag.StringVar(&format, "format", "json", "Output format (json, markdown, yaml)") flag.IntVar(&concurrency, "concurrency", runtime.NumCPU(), "Number of concurrent workers (default: number of CPU cores)") } @@ -36,116 +38,97 @@ func init() { func Run(ctx context.Context) error { flag.Parse() - if sourceDir == "" || destination == "" { - return fmt.Errorf( - "usage: gibidify " + - "-source " + - "-destination " + - "[--prefix=\"...\"] " + - "[--suffix=\"...\"] " + - "[-concurrency=]", - ) + // We need at least a source directory + if sourceDir == "" { + return fmt.Errorf("usage: gibidify -source [--destination ] [--format=json|yaml|markdown] ") + } + + // If destination is not specified, auto-generate it using the base name of sourceDir + "." + format + if destination == "" { + absRoot, err := filepath.Abs(sourceDir) + if err != nil { + return fmt.Errorf("failed to get absolute path for %s: %w", sourceDir, err) + } + baseName := filepath.Base(absRoot) + // If sourceDir ends with a slash, baseName might be "." so handle that case as needed + if baseName == "." || baseName == "" { + baseName = "output" + } + destination = baseName + "." + format } - // Load configuration using Viper. config.LoadConfig() - logrus.Infof( - "Starting gibidify. Source: %s, Destination: %s, Workers: %d", - sourceDir, - destination, - concurrency, - ) + logrus.Infof("Starting gibidify. Format: %s, Source: %s, Destination: %s, Workers: %d", format, sourceDir, destination, concurrency) - // 1. Collect files using the file walker (ProdWalker). + // Collect files files, err := fileproc.CollectFiles(sourceDir) if err != nil { return fmt.Errorf("error collecting files: %w", err) } logrus.Infof("Found %d files to process", len(files)) - // 2. Open the destination file and write the header. + // Open output file outFile, err := os.Create(destination) if err != nil { return fmt.Errorf("failed to create output file %s: %w", destination, err) } defer func(outFile *os.File) { - err := outFile.Close() - if err != nil { - logrus.Errorf("failed to close output file %s: %v", destination, err) + if err := outFile.Close(); err != nil { + logrus.Errorf("Error closing output file: %v", err) } }(outFile) - header := prefix + "\n" + - "The following text is a Git repository with code. " + - "The structure of the text are sections that begin with ----, " + - "followed by a single line containing the file path and file name, " + - "followed by a variable amount of lines containing the file contents. " + - "The text representing the Git repository ends when the symbols --END-- are encountered.\n" - - if _, err := outFile.WriteString(header); err != nil { - return fmt.Errorf("failed to write header: %w", err) - } - - // 3. Set up channels and a worker pool for processing files. + // Create channels fileCh := make(chan string) writeCh := make(chan fileproc.WriteRequest) + writerDone := make(chan struct{}) + + // Start writer goroutine + go fileproc.StartWriter(outFile, writeCh, writerDone, format, prefix, suffix) + var wg sync.WaitGroup - // Start the writer goroutine. - writerDone := make(chan struct{}) - go fileproc.StartWriter(outFile, writeCh, writerDone) - - // Start worker goroutines. + // Start worker goroutines with context cancellation for i := 0; i < concurrency; i++ { wg.Add(1) go func() { defer wg.Done() for { select { - case fp, ok := <-fileCh: + case <-ctx.Done(): + return + case filePath, ok := <-fileCh: if !ok { return } - // Process the file. - fileproc.ProcessFile(fp, writeCh, nil) - case <-ctx.Done(): - return + // Pass sourceDir to ProcessFile so it knows the 'root' + absRoot, err := filepath.Abs(sourceDir) + if err != nil { + logrus.Errorf("Failed to get absolute path for %s: %v", sourceDir, err) + return + } + fileproc.ProcessFile(filePath, writeCh, absRoot) } } }() } - // Feed file paths to the worker pool with progress bar feedback. - bar := progressbar.Default(int64(len(files))) -loop: + // Feed files to worker pool while checking for cancellation for _, fp := range files { select { - case fileCh <- fp: - _ = bar.Add(1) case <-ctx.Done(): close(fileCh) - break loop + return ctx.Err() + case fileCh <- fp: } } close(fileCh) - // Wait for all workers to finish. wg.Wait() close(writeCh) <-writerDone - // Check for context cancellation. - if err := ctx.Err(); err != nil { - return err - } - - // 4. Write footer. - footer := "--END--\n" + suffix - if _, err := outFile.WriteString(footer); err != nil { - return fmt.Errorf("failed to write footer: %w", err) - } - logrus.Infof("Processing completed. Output saved to %s", destination) return nil }