ivuorinen/gibidify
1
0
Fork 0
mirror of https://github.com/ivuorinen/gibidify.git synced 2026-02-17 12:50:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: update go to 1.25, add permissions and envs (#49)

Browse Source 
* chore(ci): update go to 1.25, add permissions and envs
* fix(ci): update pr-lint.yml
* chore: update go, fix linting
* fix: tests and linting
* fix(lint): lint fixes, renovate should now pass
* fix: updates, security upgrades
* chore: workflow updates, lint
* fix: more lint, checkmake, and other fixes
* fix: more lint, convert scripts to POSIX compliant
* fix: simplify codeql workflow
* tests: increase test coverage, fix found issues
* fix(lint): editorconfig checking, add to linters
* fix(lint): shellcheck, add to linters
* fix(lint): apply cr comment suggestions
* fix(ci): remove step-security/harden-runner
* fix(lint): remove duplication, apply cr fixes
* fix(ci): tests in CI/CD pipeline
* chore(lint): deduplication of strings
* fix(lint): apply cr comment suggestions
* fix(ci): actionlint
* fix(lint): apply cr comment suggestions
* chore: lint, add deps management
This commit is contained in:
Ismo Vuorinen
2025-10-10 12:14:42 +03:00
committed by GitHub
parent 958f5952a0
commit 3f65b813bd
100 changed files with 6997 additions and 1225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
benchmark/benchmark.go
View File

@@ -12,11 +12,11 @@ import (
"github.com/ivuorinen/gibidify/config"
"github.com/ivuorinen/gibidify/fileproc"
"github.com/ivuorinen/gibidify/utils"
"github.com/ivuorinen/gibidify/gibidiutils"
)
// BenchmarkResult represents the results of a benchmark run.
type BenchmarkResult struct {
// Result represents the results of a benchmark run.
type Result struct {
Name string
Duration time.Duration
FilesProcessed int
@@ -42,14 +42,14 @@ type CPUStats struct {
Goroutines int
}
// BenchmarkSuite represents a collection of benchmarks.
type BenchmarkSuite struct {
// Suite represents a collection of benchmarks.
type Suite struct {
Name string
Results []BenchmarkResult
Results []Result
}
// FileCollectionBenchmark benchmarks file collection operations.
func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, error) {
func FileCollectionBenchmark(sourceDir string, numFiles int) (*Result, error) {
// Load configuration to ensure proper file filtering
config.LoadConfig()
@@ -58,7 +58,12 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
if sourceDir == "" {
tempDir, cleanupFunc, err := createBenchmarkFiles(numFiles)
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create benchmark files")
return nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeFileSystem,
gibidiutils.CodeFSAccess,
"failed to create benchmark files",
)
}
cleanup = cleanupFunc
defer cleanup()
@@ -74,7 +79,12 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
// Run the file collection benchmark
files, err := fileproc.CollectFiles(sourceDir)
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "benchmark file collection failed")
return nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"benchmark file collection failed",
)
}
duration := time.Since(startTime)
@@ -91,7 +101,7 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
}
}
result := &BenchmarkResult{
result := &Result{
Name: "FileCollection",
Duration: duration,
FilesProcessed: len(files),
@@ -113,7 +123,9 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
}
// FileProcessingBenchmark benchmarks full file processing pipeline.
func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (*BenchmarkResult, error) {
//
//revive:disable-next-line:function-length
func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (*Result, error) {
// Load configuration to ensure proper file filtering
config.LoadConfig()
@@ -122,7 +134,12 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
// Create temporary directory with test files
tempDir, cleanupFunc, err := createBenchmarkFiles(100)
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create benchmark files")
return nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeFileSystem,
gibidiutils.CodeFSAccess,
"failed to create benchmark files",
)
}
cleanup = cleanupFunc
defer cleanup()
@@ -132,7 +149,12 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
// Create temporary output file
outputFile, err := os.CreateTemp("", "benchmark_output_*."+format)
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileCreate, "failed to create benchmark output file")
return nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOFileCreate,
"failed to create benchmark output file",
)
}
defer func() {
if err := outputFile.Close(); err != nil {
@@ -154,13 +176,29 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
// Run the full processing pipeline
files, err := fileproc.CollectFiles(sourceDir)
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "benchmark file collection failed")
return nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"benchmark file collection failed",
)
}
// Process files with concurrency
err = runProcessingPipeline(context.Background(), files, outputFile, format, concurrency, sourceDir)
err = runProcessingPipeline(context.Background(), processingConfig{
files: files,
outputFile: outputFile,
format: format,
concurrency: concurrency,
sourceDir: sourceDir,
})
if err != nil {
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "benchmark processing pipeline failed")
return nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingFileRead,
"benchmark processing pipeline failed",
)
}
duration := time.Since(startTime)
@@ -177,7 +215,7 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
}
}
result := &BenchmarkResult{
result := &Result{
Name: fmt.Sprintf("FileProcessing_%s_c%d", format, concurrency),
Duration: duration,
FilesProcessed: len(files),
@@ -199,16 +237,22 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
}
// ConcurrencyBenchmark benchmarks different concurrency levels.
func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []int) (*BenchmarkSuite, error) {
suite := &BenchmarkSuite{
func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []int) (*Suite, error) {
suite := &Suite{
Name: "ConcurrencyBenchmark",
Results: make([]BenchmarkResult, 0, len(concurrencyLevels)),
Results: make([]Result, 0, len(concurrencyLevels)),
}
for _, concurrency := range concurrencyLevels {
result, err := FileProcessingBenchmark(sourceDir, format, concurrency)
if err != nil {
return nil, utils.WrapErrorf(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed for level %d", concurrency)
return nil, gibidiutils.WrapErrorf(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"concurrency benchmark failed for level %d",
concurrency,
)
}
suite.Results = append(suite.Results, *result)
}
@@ -217,16 +261,22 @@ func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []i
}
// FormatBenchmark benchmarks different output formats.
func FormatBenchmark(sourceDir string, formats []string) (*BenchmarkSuite, error) {
suite := &BenchmarkSuite{
func FormatBenchmark(sourceDir string, formats []string) (*Suite, error) {
suite := &Suite{
Name: "FormatBenchmark",
Results: make([]BenchmarkResult, 0, len(formats)),
Results: make([]Result, 0, len(formats)),
}
for _, format := range formats {
result, err := FileProcessingBenchmark(sourceDir, format, runtime.NumCPU())
if err != nil {
return nil, utils.WrapErrorf(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed for format %s", format)
return nil, gibidiutils.WrapErrorf(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"format benchmark failed for format %s",
format,
)
}
suite.Results = append(suite.Results, *result)
}
@@ -238,7 +288,12 @@ func FormatBenchmark(sourceDir string, formats []string) (*BenchmarkSuite, error
func createBenchmarkFiles(numFiles int) (string, func(), error) {
tempDir, err := os.MkdirTemp("", "gibidify_benchmark_*")
if err != nil {
return "", nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create temp directory")
return "", nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeFileSystem,
gibidiutils.CodeFSAccess,
"failed to create temp directory",
)
}
cleanup := func() {
@@ -256,8 +311,15 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
{".go", "package main\n\nfunc main() {\n\tprintln(\"Hello, World!\")\n}"},
{".js", "console.log('Hello, World!');"},
{".py", "print('Hello, World!')"},
{".java", "public class Hello {\n\tpublic static void main(String[] args) {\n\t\tSystem.out.println(\"Hello, World!\");\n\t}\n}"},
{".cpp", "#include <iostream>\n\nint main() {\n\tstd::cout << \"Hello, World!\" << std::endl;\n\treturn 0;\n}"},
{
".java",
"public class Hello {\n\tpublic static void main(String[] args) {" +
"\n\t\tSystem.out.println(\"Hello, World!\");\n\t}\n}",
},
{
".cpp",
"#include <iostream>\n\nint main() {\n\tstd::cout << \"Hello, World!\" << std::endl;\n\treturn 0;\n}",
},
{".rs", "fn main() {\n\tprintln!(\"Hello, World!\");\n}"},
{".rb", "puts 'Hello, World!'"},
{".php", "<?php\necho 'Hello, World!';\n?>"},
@@ -272,9 +334,14 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
// Create subdirectories for some files
if i%10 == 0 {
subdir := filepath.Join(tempDir, fmt.Sprintf("subdir_%d", i/10))
if err := os.MkdirAll(subdir, 0o755); err != nil {
if err := os.MkdirAll(subdir, 0o750); err != nil {
cleanup()
return "", nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create subdirectory")
return "", nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeFileSystem,
gibidiutils.CodeFSAccess,
"failed to create subdirectory",
)
}
filename = filepath.Join(subdir, filename)
} else {
@@ -287,9 +354,14 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
content += fmt.Sprintf("// Line %d\n%s\n", j, fileType.content)
}
if err := os.WriteFile(filename, []byte(content), 0o644); err != nil {
if err := os.WriteFile(filename, []byte(content), 0o600); err != nil {
cleanup()
return "", nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileWrite, "failed to write benchmark file")
return "", nil, gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeIO,
gibidiutils.CodeIOFileWrite,
"failed to write benchmark file",
)
}
}
@@ -297,23 +369,41 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
}
// runProcessingPipeline runs the processing pipeline similar to main.go.
func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.File, format string, concurrency int, sourceDir string) error {
fileCh := make(chan string, concurrency)
writeCh := make(chan fileproc.WriteRequest, concurrency)
// processingConfig holds configuration for processing pipeline.
type processingConfig struct {
files []string
outputFile *os.File
format string
concurrency int
sourceDir string
}
func runProcessingPipeline(ctx context.Context, config processingConfig) error {
fileCh := make(chan string, config.concurrency)
writeCh := make(chan fileproc.WriteRequest, config.concurrency)
writerDone := make(chan struct{})
// Start writer
go fileproc.StartWriter(outputFile, writeCh, writerDone, format, "", "")
go fileproc.StartWriter(config.outputFile, writeCh, writerDone, fileproc.WriterConfig{
Format: config.format,
Prefix: "",
Suffix: "",
})
// Get absolute path once
absRoot, err := utils.GetAbsolutePath(sourceDir)
absRoot, err := gibidiutils.GetAbsolutePath(config.sourceDir)
if err != nil {
return utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSPathResolution, "failed to get absolute path for source directory")
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeFileSystem,
gibidiutils.CodeFSPathResolution,
"failed to get absolute path for source directory",
)
}
// Start workers with proper synchronization
var workersDone sync.WaitGroup
for i := 0; i < concurrency; i++ {
for i := 0; i < config.concurrency; i++ {
workersDone.Add(1)
go func() {
defer workersDone.Done()
@@ -324,7 +414,7 @@ func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.F
}
// Send files to workers
for _, file := range files {
for _, file := range config.files {
select {
case <-ctx.Done():
close(fileCh)
@@ -347,8 +437,8 @@ func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.F
return nil
}
// PrintBenchmarkResult prints a formatted benchmark result.
func PrintBenchmarkResult(result *BenchmarkResult) {
// PrintResult prints a formatted benchmark result.
func PrintResult(result *Result) {
fmt.Printf("=== %s ===\n", result.Name)
fmt.Printf("Duration: %v\n", result.Duration)
fmt.Printf("Files Processed: %d\n", result.FilesProcessed)
@@ -356,16 +446,17 @@ func PrintBenchmarkResult(result *BenchmarkResult) {
fmt.Printf("Files/sec: %.2f\n", result.FilesPerSecond)
fmt.Printf("Bytes/sec: %.2f MB/sec\n", result.BytesPerSecond/1024/1024)
fmt.Printf("Memory Usage: +%.2f MB (Sys: +%.2f MB)\n", result.MemoryUsage.AllocMB, result.MemoryUsage.SysMB)
fmt.Printf("GC Runs: %d (Pause: %v)\n", result.MemoryUsage.NumGC, time.Duration(result.MemoryUsage.PauseTotalNs))
pauseDuration := time.Duration(gibidiutils.SafeUint64ToInt64WithDefault(result.MemoryUsage.PauseTotalNs, 0))
fmt.Printf("GC Runs: %d (Pause: %v)\n", result.MemoryUsage.NumGC, pauseDuration)
fmt.Printf("Goroutines: %d\n", result.CPUUsage.Goroutines)
fmt.Println()
}
// PrintBenchmarkSuite prints all results in a benchmark suite.
func PrintBenchmarkSuite(suite *BenchmarkSuite) {
// PrintSuite prints all results in a benchmark suite.
func PrintSuite(suite *Suite) {
fmt.Printf("=== %s ===\n", suite.Name)
for _, result := range suite.Results {
PrintBenchmarkResult(&result)
for i := range suite.Results {
PrintResult(&suite.Results[i])
}
}
@@ -380,26 +471,41 @@ func RunAllBenchmarks(sourceDir string) error {
fmt.Println("Running file collection benchmark...")
result, err := FileCollectionBenchmark(sourceDir, 1000)
if err != nil {
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file collection benchmark failed")
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"file collection benchmark failed",
)
}
PrintBenchmarkResult(result)
PrintResult(result)
// Format benchmarks
fmt.Println("Running format benchmarks...")
formatSuite, err := FormatBenchmark(sourceDir, []string{"json", "yaml", "markdown"})
if err != nil {
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed")
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"format benchmark failed",
)
}
PrintBenchmarkSuite(formatSuite)
PrintSuite(formatSuite)
// Concurrency benchmarks
fmt.Println("Running concurrency benchmarks...")
concurrencyLevels := []int{1, 2, 4, 8, runtime.NumCPU()}
concurrencySuite, err := ConcurrencyBenchmark(sourceDir, "json", concurrencyLevels)
if err != nil {
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed")
return gibidiutils.WrapError(
err,
gibidiutils.ErrorTypeProcessing,
gibidiutils.CodeProcessingCollection,
"concurrency benchmark failed",
)
}
PrintBenchmarkSuite(concurrencySuite)
PrintSuite(concurrencySuite)
return nil
}