diff --git a/.checkmake b/.checkmake index f050ebd..fb06f79 100644 --- a/.checkmake +++ b/.checkmake @@ -1,8 +1,14 @@ # checkmake configuration -# See: https://github.com/mrtazz/checkmake#configuration +# See: https://github.com/checkmake/checkmake#configuration [rules.timestampexpansion] disabled = true [rules.maxbodylength] -disabled = true \ No newline at end of file +disabled = true + +[rules.minphony] +disabled = true + +[rules.phonydeclared] +disabled = true diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1b26c47 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,38 @@ +# Git +.git +.github +.gitignore + +# Build artifacts +gibidify +gibidify-* +dist/ +coverage.out +coverage.html +test-results.json +*.sarif + +# Documentation +*.md +docs/ + +# Config and tooling +.checkmake +.editorconfig +.golangci.yml +.yamllint +revive.toml + +# Scripts +scripts/ + +# IDE +.vscode +.idea +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/.editorconfig b/.editorconfig index 82c5a83..34522aa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,19 +8,26 @@ indent_size = 2 indent_style = tab tab_width = 2 -[*.yml] -indent_style = space +[*.go] +max_line_length = 120 [*.md] trim_trailing_whitespace = false -[*.{yml,yaml,json}] +[*.{yml,yaml,json,toml}] indent_style = space max_line_length = 250 +[*.{yaml.example,yml.example}] +indent_style = space + +[.yamllint] +indent_style = space + [LICENSE] max_line_length = 80 indent_size = 0 indent_style = space -trim_trailing_whitespace = true +[Makefile] +max_line_length = 80 diff --git a/.editorconfig-checker.json b/.editorconfig-checker.json new file mode 100644 index 0000000..0382392 --- /dev/null +++ b/.editorconfig-checker.json @@ -0,0 +1,14 @@ +{ + "Exclude": [".git", "vendor", "node_modules", "README\\.md"], + "AllowedContentTypes": [], + "PassedFiles": [], + "Disable": { + "IndentSize": false, + "EndOfLine": false, + "InsertFinalNewline": false, + "TrimTrailingWhitespace": false, + "MaxLineLength": false + }, + "SpacesAfterTabs": false, + "NoColor": false +} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000..9774a75 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,15 @@ +name: "Setup Go with Runner Hardening" +description: "Reusable action to set up Go" +inputs: + token: + description: "GitHub token for checkout (optional)" + required: false + default: "" +runs: + using: "composite" + steps: + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version-file: "go.mod" + cache: true diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index d72eed6..2678c1f 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -9,8 +9,7 @@ on: release: types: [created] -permissions: - contents: read +permissions: {} jobs: test: @@ -25,51 +24,60 @@ jobs: statuses: write steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - with: - egress-policy: audit - - - name: Checkout code + - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Set up Go - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + - name: Setup Go + uses: ./.github/actions/setup with: - go-version-file: "./go.mod" - cache: true + token: ${{ github.token }} - - name: Install dependencies - run: go mod tidy + - name: Download dependencies + shell: bash + run: go mod download - - name: Run tests - run: go test -json ./... > test-results.json - - - name: Generate coverage report - run: go test -coverprofile=coverage.out ./... + - name: Run tests with coverage + shell: bash + run: | + go test -race -covermode=atomic -json -coverprofile=coverage.out ./... | tee test-results.json - name: Check coverage id: coverage + if: always() + shell: bash run: | + if [[ ! -f coverage.out ]]; then + echo "coverage.out is missing; tests likely failed before producing coverage" + exit 1 + fi coverage="$(go tool cover -func=coverage.out | grep total | awk '{print substr($3, 1, length($3)-1)}')" echo "total_coverage=$coverage" >> "$GITHUB_ENV" echo "Coverage: $coverage%" - name: Upload test results + if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: test-results path: test-results.json - name: Cleanup - run: rm coverage.out + if: always() + shell: bash + run: rm -f coverage.out test-results.json - name: Fail if coverage is below threshold + if: always() + shell: bash run: | - if (( $(echo "$total_coverage < 50" | bc -l) )); then - echo "Coverage ($total_coverage%) is below the threshold (50%)" + if [[ -z "${total_coverage:-}" ]]; then + echo "total_coverage is unset; previous step likely failed" exit 1 fi + awk -v cov="$total_coverage" 'BEGIN{ if (cov < 60) exit 1; else exit 0 }' || { + echo "Coverage ($total_coverage%) is below the threshold (60%)" + exit 1 + } build: name: Build Binaries @@ -89,13 +97,13 @@ jobs: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Set up Go - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + - name: Setup Go + uses: ./.github/actions/setup with: - go-version-file: "./go.mod" + token: ${{ github.token }} - - name: Run go mod tidy - run: go mod tidy + - name: Download dependencies + run: go mod download - name: Build binary for ${{ matrix.goos }}-${{ matrix.goarch }} run: | @@ -132,24 +140,24 @@ jobs: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Download Linux binaries - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + - name: Setup Go + uses: ./.github/actions/setup with: - name: gibidify-linux-amd64 - path: . + token: ${{ github.token }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Log in to GitHub Container Registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Build and push multi-arch Docker image run: | - chmod +x gibidify-linux-amd64 - mv gibidify-linux-amd64 gibidify - docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 \ - --tag ghcr.io/${{ github.repository }}/gibidify:${{ github.ref_name }} \ - --tag ghcr.io/${{ github.repository }}/gibidify:latest \ - --push \ - --squash . + echo "${{ github.token }}" | docker login ghcr.io \ + -u "$(echo "${{ github.actor }}" | tr '[:upper:]' '[:lower:]')" \ + --password-stdin + + - name: Build and push Docker image + run: | + repo="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + docker buildx build --platform linux/amd64 \ + --tag "ghcr.io/${repo}/gibidify:${{ github.ref_name }}" \ + --tag "ghcr.io/${repo}/gibidify:latest" \ + --push . diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..1bb9c14 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,39 @@ +name: CodeQL Analysis + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +permissions: {} + +jobs: + analyze: + name: Analyze Code + runs-on: ubuntu-latest + + permissions: + security-events: write + contents: read + actions: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Setup Go + uses: ./.github/actions/setup + with: + token: ${{ github.token }} + + - name: Initialize CodeQL + uses: github/codeql-action/init@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 + with: + languages: go + + - name: Autobuild + uses: github/codeql-action/autobuild@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/pr-lint.yml index 82e5179..e49a0ab 100644 --- a/.github/workflows/pr-lint.yml +++ b/.github/workflows/pr-lint.yml @@ -9,7 +9,7 @@ on: pull_request: branches: [master, main] -permissions: read-all +permissions: {} jobs: Linter: @@ -21,7 +21,12 @@ jobs: pull-requests: write statuses: write steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Setup Go + uses: ./.github/actions/setup with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ github.token }} + - uses: ivuorinen/actions/pr-lint@dc895c40ffdce61ab057fb992f4e00f1efdcbcbf # 25.10.7 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 42e0c35..ccc9278 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -7,45 +7,37 @@ on: branches: [main, develop] schedule: # Run security scan weekly on Sundays at 00:00 UTC - - cron: '0 0 * * 0' + - cron: "0 0 * * 0" -permissions: - security-events: write - contents: read - actions: read +permissions: {} jobs: security: name: Security Analysis runs-on: ubuntu-latest + permissions: + security-events: write + contents: read + actions: read + steps: - - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Setup Go - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6 + uses: ./.github/actions/setup with: - go-version: '1.23' - - - name: Cache Go modules - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + token: ${{ github.token }} # Security Scanning with gosec - name: Run gosec Security Scanner uses: securego/gosec@15d5c61e866bc2e2e8389376a31f1e5e09bde7d8 # v2.22.9 with: - args: '-fmt sarif -out gosec-results.sarif ./...' + args: "-fmt sarif -out gosec-results.sarif ./..." - name: Upload gosec results to GitHub Security tab - uses: github/codeql-action/upload-sarif@df559355d593797519d70b90fc8edd5db049e7a2 # v3 + uses: github/codeql-action/upload-sarif@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7 if: always() with: sarif_file: gosec-results.sarif @@ -60,24 +52,17 @@ jobs: run: | if [ -s govulncheck-results.json ]; then echo "::warning::Vulnerability check completed. Check govulncheck-results.json for details." - if grep -q '"finding"' govulncheck-results.json; then + if grep -i -q '"finding"' govulncheck-results.json; then echo "::error::Vulnerabilities found in dependencies!" cat govulncheck-results.json exit 1 fi fi - # Additional Security Linting - - name: Run security-focused golangci-lint - run: | - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest - golangci-lint run --enable=gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused \ - --timeout=5m - # Makefile Linting - name: Run checkmake on Makefile run: | - go install github.com/mrtazz/checkmake/cmd/checkmake@latest + go install github.com/checkmake/checkmake/cmd/checkmake@latest checkmake --config=.checkmake Makefile # Shell Script Formatting Check @@ -86,27 +71,11 @@ jobs: go install mvdan.cc/sh/v3/cmd/shfmt@latest shfmt -d . - # YAML Linting - name: Run YAML linting - run: | - go install github.com/excilsploft/yamllint@latest - yamllint -c .yamllint . - - # Secrets Detection (basic patterns) - - name: Run secrets detection - run: | - echo "Scanning for potential secrets..." - # Look for common secret patterns - git log --all --full-history -- . | grep -i -E "(password|secret|key|token|api_key)" || true - find . -type f -name "*.go" -exec grep -H -i -E "(password|secret|key|token|api_key)\s*[:=]" {} \; || true - - # Check for hardcoded IPs and URLs - - name: Check for hardcoded network addresses - run: | - echo "Scanning for hardcoded network addresses..." - find . -type f -name "*.go" -exec grep -H -E "([0-9]{1,3}\.){3}[0-9]{1,3}" {} \; || true - find . -type f -name "*.go" -exec grep -H -E "https?://[^/\s]+" {} \; | \ - grep -v "example.com|localhost|127.0.0.1" || true + uses: ibiqlik/action-yamllint@2576378a8e339169678f9939646ee3ee325e845c # v3.1.1 + with: + file_or_dir: . + strict: true # Docker Security (if Dockerfile exists) - name: Run Docker security scan @@ -115,24 +84,9 @@ jobs: docker run --rm -v "$PWD":/workspace \ aquasec/trivy:latest fs --security-checks vuln,config /workspace/Dockerfile || true - # SAST with CodeQL (if available) - - name: Initialize CodeQL - if: github.event_name != 'schedule' - uses: github/codeql-action/init@df559355d593797519d70b90fc8edd5db049e7a2 # v3 - with: - languages: go - - - name: Autobuild - if: github.event_name != 'schedule' - uses: github/codeql-action/autobuild@df559355d593797519d70b90fc8edd5db049e7a2 # v3 - - - name: Perform CodeQL Analysis - if: github.event_name != 'schedule' - uses: github/codeql-action/analyze@df559355d593797519d70b90fc8edd5db049e7a2 # v3 - # Upload artifacts for review - name: Upload security scan results - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: security-scan-results diff --git a/.gitignore b/.gitignore index 6c9662f..61f96d4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,8 @@ megalinter-reports/* coverage.* *.out gibidify-benchmark +gosec-report.json +gosec-results.sarif +govulncheck-report.json +govulncheck-errors.log +security-report.md diff --git a/.go-version b/.go-version index a6c2798..d905a6d 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.23.0 +1.25.1 diff --git a/.mega-linter.yml b/.mega-linter.yml index 2447b22..d10e151 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -15,6 +15,9 @@ PRINT_ALPACA: false # Print Alpaca logo in console SARIF_REPORTER: true # Generate SARIF report SHOW_SKIPPED_LINTERS: false # Show skipped linters in MegaLinter log +GO_REVIVE_CLI_LINT_MODE: project + DISABLE_LINTERS: - REPOSITORY_DEVSKIM - REPOSITORY_TRIVY + - GO_GOLANGCI_LINT diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09b6050..2dfeb53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: golangci-lint args: ["--timeout=5m"] - repo: https://github.com/tekwizely/pre-commit-golang - rev: v1.0.0-rc.1 + rev: v1.0.0-rc.2 hooks: - id: go-build-mod alias: build @@ -13,3 +13,12 @@ repos: alias: tidy - id: go-fmt alias: fmt + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: 3.4.0 + hooks: + - id: editorconfig-checker + alias: ec + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.11.0.1 + hooks: + - id: shellcheck diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 0000000..14d86ad --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1 @@ +/cache diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 0000000..9246931 --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,73 @@ +# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby) +# * For C, use cpp +# * For JavaScript, use typescript +# Special requirements: +# * csharp: Requires the presence of a .sln file in the project folder. +language: go + +# whether to use the project's gitignore file to ignore files +# Added on 2025-04-07 +ignore_all_files_in_gitignore: true +# list of additional paths to ignore +# same syntax as gitignore, so you can use * and ** +# Was previously called `ignored_dirs`, please update your config if you are using that. +# Added (renamed) on 2025-04-07 +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + +# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. +# Below is the complete list of tools for convenience. +# To make sure you have the latest list of tools, and to view their descriptions, +# execute `uv run scripts/print_tool_overview.py`. +# +# * `activate_project`: Activates a project by name. +# * `check_onboarding_performed`: Checks whether project onboarding was already performed. +# * `create_text_file`: Creates/overwrites a file in the project directory. +# * `delete_lines`: Deletes a range of lines within a file. +# * `delete_memory`: Deletes a memory from Serena's project-specific memory store. +# * `execute_shell_command`: Executes a shell command. +# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced. +# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location +# (optionally filtered by type). +# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given +# name/substring (optionally filtered by type). +# * `get_current_config`: Prints the current configuration of the agent, including the active +# and available projects, tools, contexts, and modes. +# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. +# * `initial_instructions`: Gets the initial instructions for the current project. +# Should only be used in settings where the system prompt cannot be set, +# e.g. in clients you have no control over, like Claude Desktop. +# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +# * `insert_at_line`: Inserts content at a given line in a file. +# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). +# * `list_memories`: Lists memories in Serena's project-specific memory store. +# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, +# e.g. for testing or building). +# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation +# (in order to continue with the necessary context). +# * `read_file`: Reads a file within the project directory. +# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. +# * `remove_project`: Removes a project from the Serena configuration. +# * `replace_lines`: Replaces a range of lines within a file with new content. +# * `replace_symbol_body`: Replaces the full definition of a symbol. +# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. +# * `search_for_pattern`: Performs a search for a pattern in the project. +# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. +# * `switch_modes`: Activates modes by providing a list of their names +# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. +# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on +# track with the current task. +# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. +# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. +excluded_tools: [] + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +project_name: "gibidify" diff --git a/CLAUDE.md b/CLAUDE.md index fb9af0b..dc90940 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,4 +44,7 @@ EditorConfig (LF, tabs), semantic commits, testing required ## Workflow -1. `make lint-fix` first 2. >80% coverage 3. Follow patterns 4. Update docs +1. `make lint-fix` first +2. >80% coverage +3. Follow patterns +4. Update docs diff --git a/Dockerfile b/Dockerfile index b7672eb..ab54270 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,38 @@ -# Use a minimal base image +# Build stage - builds the binary for the target architecture +FROM --platform=$BUILDPLATFORM golang:1.25.1-alpine AS builder + +# Build arguments automatically set by buildx +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT + +WORKDIR /build + +# Copy go mod files first for better layer caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the binary for the target platform +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -ldflags="-s -w" -o gibidify . + +# Runtime stage - minimal image with the binary FROM alpine:3.22.1 -# Add user -RUN useradd -ms /bin/bash gibidify +# Install ca-certificates for HTTPS and create non-root user +# hadolint ignore=DL3018 +# kics-scan ignore-line +RUN apk add --no-cache ca-certificates && \ + adduser -D -s /bin/sh gibidify -# Use the new user +# Copy the binary from builder +COPY --from=builder /build/gibidify /usr/local/bin/gibidify + +# Use non-root user USER gibidify -# Copy the gibidify binary into the container -COPY gibidify /usr/local/bin/gibidify - -# Ensure the binary is executable -RUN chmod +x /usr/local/bin/gibidify - # Set the entrypoint ENTRYPOINT ["/usr/local/bin/gibidify"] diff --git a/Makefile b/Makefile index 7241ae6..e9f2873 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,8 @@ -.PHONY: help install-tools lint lint-fix lint-verbose test coverage build clean all build-benchmark benchmark benchmark-collection benchmark-processing benchmark-concurrency benchmark-format security security-full vuln-check check-all dev-setup +.PHONY: all clean test test-coverage build coverage help lint lint-fix \ + lint-verbose install-tools benchmark benchmark-collection \ + benchmark-concurrency benchmark-format benchmark-processing \ + build-benchmark check-all ci-lint ci-test dev-setup security \ + security-full vuln-check deps-update deps-check deps-tidy # Default target shows help .DEFAULT_GOAL := help @@ -6,7 +10,7 @@ # All target runs full workflow all: lint test build -# Help target +# Help target help: @cat scripts/help.txt @@ -16,6 +20,8 @@ install-tools: @go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest @echo "Installing gofumpt..." @go install mvdan.cc/gofumpt@latest + @echo "Installing golines..." + @go install github.com/segmentio/golines@latest @echo "Installing goimports..." @go install golang.org/x/tools/cmd/goimports@latest @echo "Installing staticcheck..." @@ -24,12 +30,19 @@ install-tools: @go install github.com/securego/gosec/v2/cmd/gosec@latest @echo "Installing gocyclo..." @go install github.com/fzipp/gocyclo/cmd/gocyclo@latest + @echo "Installing revive..." + @go install github.com/mgechev/revive@latest @echo "Installing checkmake..." - @go install github.com/mrtazz/checkmake/cmd/checkmake@latest + @go install github.com/checkmake/checkmake/cmd/checkmake@latest + @echo "Installing shellcheck..." + @go install github.com/koalaman/shellcheck/cmd/shellcheck@latest @echo "Installing shfmt..." @go install mvdan.cc/sh/v3/cmd/shfmt@latest @echo "Installing yamllint (Go-based)..." @go install github.com/excilsploft/yamllint@latest + @echo "Installing editorconfig-checker..." + @go install github.com/editorconfig-checker/editorconfig-checker/\ + cmd/editorconfig-checker@latest @echo "All tools installed successfully!" # Run linters @@ -40,6 +53,8 @@ lint: lint-fix: @echo "Running gofumpt..." @gofumpt -l -w . + @echo "Running golines..." + @golines -w -m 120 --base-formatter="gofumpt" --shorten-comments . @echo "Running goimports..." @goimports -w -local github.com/ivuorinen/gibidify . @echo "Running go fmt..." @@ -47,32 +62,46 @@ lint-fix: @echo "Running go mod tidy..." @go mod tidy @echo "Running shfmt formatting..." - @shfmt -w -i 2 -ci . + @shfmt -w -i 0 -ci . @echo "Running golangci-lint with --fix..." @golangci-lint run --fix ./... @echo "Auto-fix completed. Running final lint check..." @golangci-lint run ./... + @echo "Running revive..." + @revive -config revive.toml -formatter friendly ./... @echo "Running checkmake..." @checkmake --config=.checkmake Makefile @echo "Running yamllint..." - @yamllint -c .yamllint . + @yamllint . # Run linters with verbose output lint-verbose: @echo "Running golangci-lint (verbose)..." @golangci-lint run -v ./... @echo "Running checkmake (verbose)..." - @checkmake --config=.checkmake --format="{{.Line}}:{{.Rule}}:{{.Violation}}" Makefile + @checkmake --config=.checkmake \ + --format="{{.Line}}:{{.Rule}}:{{.Violation}}" Makefile @echo "Running shfmt check (verbose)..." @shfmt -d . @echo "Running yamllint (verbose)..." - @yamllint -c .yamllint -f parsable . + @yamllint . # Run tests test: @echo "Running tests..." @go test -race -v ./... +# Run tests with coverage output +test-coverage: + @echo "Running tests with coverage..." + @go test -race -v -coverprofile=coverage.out -covermode=atomic ./... + @echo "" + @echo "Coverage summary:" + @go tool cover -func=coverage.out | grep total: + @echo "" + @echo "Full coverage report saved to: coverage.out" + @echo "To view HTML report, run: make coverage" + # Run tests with coverage coverage: @echo "Running tests with coverage..." @@ -94,8 +123,6 @@ clean: @echo "Clean complete" # CI-specific targets -.PHONY: ci-lint ci-test - ci-lint: @golangci-lint run --out-format=github-actions ./... @@ -138,10 +165,34 @@ security: security-full: @echo "Running full security analysis..." @./scripts/security-scan.sh - @echo "Running additional security checks..." - @golangci-lint run --enable-all --disable=depguard,exhaustruct,ireturn,varnamelen,wrapcheck --timeout=10m vuln-check: @echo "Checking for dependency vulnerabilities..." @go install golang.org/x/vuln/cmd/govulncheck@latest - @govulncheck ./... \ No newline at end of file + @govulncheck ./... + +# Dependency management targets +deps-check: + @echo "Checking for available dependency updates..." + @echo "" + @echo "Direct dependencies:" + @go list -u -m all | grep -v "indirect" | column -t + @echo "" + @echo "Note: Run 'make deps-update' to update all dependencies" + +deps-update: + @echo "Updating all dependencies to latest versions..." + @go get -u ./... + @go mod tidy + @echo "" + @echo "Dependencies updated successfully!" + @echo "Running tests to verify compatibility..." + @go test ./... + @echo "" + @echo "Update complete. Run 'make lint-fix && make test' to verify." + +deps-tidy: + @echo "Cleaning up dependencies..." + @go mod tidy + @go mod verify + @echo "Dependencies cleaned and verified successfully!" diff --git a/README.md b/README.md index 50184cc..4e151ae 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,15 @@ go build -o gibidify . ```bash ./gibidify \ - -source \ - -destination \ - -format markdown|json|yaml \ - -concurrency \ - --prefix="..." \ - --suffix="..." \ - --no-colors \ - --no-progress \ - --verbose + -source \ + -destination \ + -format markdown|json|yaml \ + -concurrency \ + --prefix="..." \ + --suffix="..." \ + --no-colors \ + --no-progress \ + --verbose ``` Flags: diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go index 6d825b7..f183dc7 100644 --- a/benchmark/benchmark.go +++ b/benchmark/benchmark.go @@ -12,11 +12,11 @@ import ( "github.com/ivuorinen/gibidify/config" "github.com/ivuorinen/gibidify/fileproc" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) -// BenchmarkResult represents the results of a benchmark run. -type BenchmarkResult struct { +// Result represents the results of a benchmark run. +type Result struct { Name string Duration time.Duration FilesProcessed int @@ -42,14 +42,14 @@ type CPUStats struct { Goroutines int } -// BenchmarkSuite represents a collection of benchmarks. -type BenchmarkSuite struct { +// Suite represents a collection of benchmarks. +type Suite struct { Name string - Results []BenchmarkResult + Results []Result } // FileCollectionBenchmark benchmarks file collection operations. -func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, error) { +func FileCollectionBenchmark(sourceDir string, numFiles int) (*Result, error) { // Load configuration to ensure proper file filtering config.LoadConfig() @@ -58,7 +58,12 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, if sourceDir == "" { tempDir, cleanupFunc, err := createBenchmarkFiles(numFiles) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create benchmark files") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSAccess, + "failed to create benchmark files", + ) } cleanup = cleanupFunc defer cleanup() @@ -74,7 +79,12 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, // Run the file collection benchmark files, err := fileproc.CollectFiles(sourceDir) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "benchmark file collection failed") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "benchmark file collection failed", + ) } duration := time.Since(startTime) @@ -91,7 +101,7 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, } } - result := &BenchmarkResult{ + result := &Result{ Name: "FileCollection", Duration: duration, FilesProcessed: len(files), @@ -113,7 +123,9 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, } // FileProcessingBenchmark benchmarks full file processing pipeline. -func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (*BenchmarkResult, error) { +// +//revive:disable-next-line:function-length +func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (*Result, error) { // Load configuration to ensure proper file filtering config.LoadConfig() @@ -122,7 +134,12 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) ( // Create temporary directory with test files tempDir, cleanupFunc, err := createBenchmarkFiles(100) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create benchmark files") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSAccess, + "failed to create benchmark files", + ) } cleanup = cleanupFunc defer cleanup() @@ -132,7 +149,12 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) ( // Create temporary output file outputFile, err := os.CreateTemp("", "benchmark_output_*."+format) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileCreate, "failed to create benchmark output file") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileCreate, + "failed to create benchmark output file", + ) } defer func() { if err := outputFile.Close(); err != nil { @@ -154,13 +176,29 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) ( // Run the full processing pipeline files, err := fileproc.CollectFiles(sourceDir) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "benchmark file collection failed") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "benchmark file collection failed", + ) } // Process files with concurrency - err = runProcessingPipeline(context.Background(), files, outputFile, format, concurrency, sourceDir) + err = runProcessingPipeline(context.Background(), processingConfig{ + files: files, + outputFile: outputFile, + format: format, + concurrency: concurrency, + sourceDir: sourceDir, + }) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "benchmark processing pipeline failed") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingFileRead, + "benchmark processing pipeline failed", + ) } duration := time.Since(startTime) @@ -177,7 +215,7 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) ( } } - result := &BenchmarkResult{ + result := &Result{ Name: fmt.Sprintf("FileProcessing_%s_c%d", format, concurrency), Duration: duration, FilesProcessed: len(files), @@ -199,16 +237,22 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) ( } // ConcurrencyBenchmark benchmarks different concurrency levels. -func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []int) (*BenchmarkSuite, error) { - suite := &BenchmarkSuite{ +func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []int) (*Suite, error) { + suite := &Suite{ Name: "ConcurrencyBenchmark", - Results: make([]BenchmarkResult, 0, len(concurrencyLevels)), + Results: make([]Result, 0, len(concurrencyLevels)), } for _, concurrency := range concurrencyLevels { result, err := FileProcessingBenchmark(sourceDir, format, concurrency) if err != nil { - return nil, utils.WrapErrorf(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed for level %d", concurrency) + return nil, gibidiutils.WrapErrorf( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "concurrency benchmark failed for level %d", + concurrency, + ) } suite.Results = append(suite.Results, *result) } @@ -217,16 +261,22 @@ func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []i } // FormatBenchmark benchmarks different output formats. -func FormatBenchmark(sourceDir string, formats []string) (*BenchmarkSuite, error) { - suite := &BenchmarkSuite{ +func FormatBenchmark(sourceDir string, formats []string) (*Suite, error) { + suite := &Suite{ Name: "FormatBenchmark", - Results: make([]BenchmarkResult, 0, len(formats)), + Results: make([]Result, 0, len(formats)), } for _, format := range formats { result, err := FileProcessingBenchmark(sourceDir, format, runtime.NumCPU()) if err != nil { - return nil, utils.WrapErrorf(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed for format %s", format) + return nil, gibidiutils.WrapErrorf( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "format benchmark failed for format %s", + format, + ) } suite.Results = append(suite.Results, *result) } @@ -238,7 +288,12 @@ func FormatBenchmark(sourceDir string, formats []string) (*BenchmarkSuite, error func createBenchmarkFiles(numFiles int) (string, func(), error) { tempDir, err := os.MkdirTemp("", "gibidify_benchmark_*") if err != nil { - return "", nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create temp directory") + return "", nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSAccess, + "failed to create temp directory", + ) } cleanup := func() { @@ -256,8 +311,15 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) { {".go", "package main\n\nfunc main() {\n\tprintln(\"Hello, World!\")\n}"}, {".js", "console.log('Hello, World!');"}, {".py", "print('Hello, World!')"}, - {".java", "public class Hello {\n\tpublic static void main(String[] args) {\n\t\tSystem.out.println(\"Hello, World!\");\n\t}\n}"}, - {".cpp", "#include \n\nint main() {\n\tstd::cout << \"Hello, World!\" << std::endl;\n\treturn 0;\n}"}, + { + ".java", + "public class Hello {\n\tpublic static void main(String[] args) {" + + "\n\t\tSystem.out.println(\"Hello, World!\");\n\t}\n}", + }, + { + ".cpp", + "#include \n\nint main() {\n\tstd::cout << \"Hello, World!\" << std::endl;\n\treturn 0;\n}", + }, {".rs", "fn main() {\n\tprintln!(\"Hello, World!\");\n}"}, {".rb", "puts 'Hello, World!'"}, {".php", ""}, @@ -272,9 +334,14 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) { // Create subdirectories for some files if i%10 == 0 { subdir := filepath.Join(tempDir, fmt.Sprintf("subdir_%d", i/10)) - if err := os.MkdirAll(subdir, 0o755); err != nil { + if err := os.MkdirAll(subdir, 0o750); err != nil { cleanup() - return "", nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create subdirectory") + return "", nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSAccess, + "failed to create subdirectory", + ) } filename = filepath.Join(subdir, filename) } else { @@ -287,9 +354,14 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) { content += fmt.Sprintf("// Line %d\n%s\n", j, fileType.content) } - if err := os.WriteFile(filename, []byte(content), 0o644); err != nil { + if err := os.WriteFile(filename, []byte(content), 0o600); err != nil { cleanup() - return "", nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileWrite, "failed to write benchmark file") + return "", nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileWrite, + "failed to write benchmark file", + ) } } @@ -297,23 +369,41 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) { } // runProcessingPipeline runs the processing pipeline similar to main.go. -func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.File, format string, concurrency int, sourceDir string) error { - fileCh := make(chan string, concurrency) - writeCh := make(chan fileproc.WriteRequest, concurrency) +// processingConfig holds configuration for processing pipeline. +type processingConfig struct { + files []string + outputFile *os.File + format string + concurrency int + sourceDir string +} + +func runProcessingPipeline(ctx context.Context, config processingConfig) error { + fileCh := make(chan string, config.concurrency) + writeCh := make(chan fileproc.WriteRequest, config.concurrency) writerDone := make(chan struct{}) // Start writer - go fileproc.StartWriter(outputFile, writeCh, writerDone, format, "", "") + go fileproc.StartWriter(config.outputFile, writeCh, writerDone, fileproc.WriterConfig{ + Format: config.format, + Prefix: "", + Suffix: "", + }) // Get absolute path once - absRoot, err := utils.GetAbsolutePath(sourceDir) + absRoot, err := gibidiutils.GetAbsolutePath(config.sourceDir) if err != nil { - return utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSPathResolution, "failed to get absolute path for source directory") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSPathResolution, + "failed to get absolute path for source directory", + ) } // Start workers with proper synchronization var workersDone sync.WaitGroup - for i := 0; i < concurrency; i++ { + for i := 0; i < config.concurrency; i++ { workersDone.Add(1) go func() { defer workersDone.Done() @@ -324,7 +414,7 @@ func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.F } // Send files to workers - for _, file := range files { + for _, file := range config.files { select { case <-ctx.Done(): close(fileCh) @@ -347,8 +437,8 @@ func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.F return nil } -// PrintBenchmarkResult prints a formatted benchmark result. -func PrintBenchmarkResult(result *BenchmarkResult) { +// PrintResult prints a formatted benchmark result. +func PrintResult(result *Result) { fmt.Printf("=== %s ===\n", result.Name) fmt.Printf("Duration: %v\n", result.Duration) fmt.Printf("Files Processed: %d\n", result.FilesProcessed) @@ -356,16 +446,17 @@ func PrintBenchmarkResult(result *BenchmarkResult) { fmt.Printf("Files/sec: %.2f\n", result.FilesPerSecond) fmt.Printf("Bytes/sec: %.2f MB/sec\n", result.BytesPerSecond/1024/1024) fmt.Printf("Memory Usage: +%.2f MB (Sys: +%.2f MB)\n", result.MemoryUsage.AllocMB, result.MemoryUsage.SysMB) - fmt.Printf("GC Runs: %d (Pause: %v)\n", result.MemoryUsage.NumGC, time.Duration(result.MemoryUsage.PauseTotalNs)) + pauseDuration := time.Duration(gibidiutils.SafeUint64ToInt64WithDefault(result.MemoryUsage.PauseTotalNs, 0)) + fmt.Printf("GC Runs: %d (Pause: %v)\n", result.MemoryUsage.NumGC, pauseDuration) fmt.Printf("Goroutines: %d\n", result.CPUUsage.Goroutines) fmt.Println() } -// PrintBenchmarkSuite prints all results in a benchmark suite. -func PrintBenchmarkSuite(suite *BenchmarkSuite) { +// PrintSuite prints all results in a benchmark suite. +func PrintSuite(suite *Suite) { fmt.Printf("=== %s ===\n", suite.Name) - for _, result := range suite.Results { - PrintBenchmarkResult(&result) + for i := range suite.Results { + PrintResult(&suite.Results[i]) } } @@ -380,26 +471,41 @@ func RunAllBenchmarks(sourceDir string) error { fmt.Println("Running file collection benchmark...") result, err := FileCollectionBenchmark(sourceDir, 1000) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file collection benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "file collection benchmark failed", + ) } - PrintBenchmarkResult(result) + PrintResult(result) // Format benchmarks fmt.Println("Running format benchmarks...") formatSuite, err := FormatBenchmark(sourceDir, []string{"json", "yaml", "markdown"}) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "format benchmark failed", + ) } - PrintBenchmarkSuite(formatSuite) + PrintSuite(formatSuite) // Concurrency benchmarks fmt.Println("Running concurrency benchmarks...") concurrencyLevels := []int{1, 2, 4, 8, runtime.NumCPU()} concurrencySuite, err := ConcurrencyBenchmark(sourceDir, "json", concurrencyLevels) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "concurrency benchmark failed", + ) } - PrintBenchmarkSuite(concurrencySuite) + PrintSuite(concurrencySuite) return nil } diff --git a/cli/errors.go b/cli/errors.go index b1e6ed3..29ca767 100644 --- a/cli/errors.go +++ b/cli/errors.go @@ -1,3 +1,4 @@ +// Package cli provides command-line interface utilities for gibidify. package cli import ( @@ -6,7 +7,7 @@ import ( "path/filepath" "strings" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // ErrorFormatter handles CLI-friendly error formatting with suggestions. @@ -19,6 +20,11 @@ func NewErrorFormatter(ui *UIManager) *ErrorFormatter { return &ErrorFormatter{ui: ui} } +// Suggestion messages for error formatting. +const ( + suggestionCheckPermissions = " %s Check file/directory permissions\n" +) + // FormatError formats an error with context and suggestions. func (ef *ErrorFormatter) FormatError(err error) { if err == nil { @@ -26,7 +32,8 @@ func (ef *ErrorFormatter) FormatError(err error) { } // Handle structured errors - if structErr, ok := err.(*utils.StructuredError); ok { + var structErr *gibidiutils.StructuredError + if errors.As(err, &structErr) { ef.formatStructuredError(structErr) return } @@ -36,12 +43,12 @@ func (ef *ErrorFormatter) FormatError(err error) { } // formatStructuredError formats a structured error with context and suggestions. -func (ef *ErrorFormatter) formatStructuredError(err *utils.StructuredError) { +func (ef *ErrorFormatter) formatStructuredError(err *gibidiutils.StructuredError) { // Print main error ef.ui.PrintError("Error: %s", err.Message) // Print error type and code - if err.Type != utils.ErrorTypeUnknown || err.Code != "" { + if err.Type != gibidiutils.ErrorTypeUnknown || err.Code != "" { ef.ui.PrintInfo("Type: %s, Code: %s", err.Type.String(), err.Code) } @@ -69,15 +76,15 @@ func (ef *ErrorFormatter) formatGenericError(err error) { } // provideSuggestions provides helpful suggestions based on the error. -func (ef *ErrorFormatter) provideSuggestions(err *utils.StructuredError) { +func (ef *ErrorFormatter) provideSuggestions(err *gibidiutils.StructuredError) { switch err.Type { - case utils.ErrorTypeFileSystem: + case gibidiutils.ErrorTypeFileSystem: ef.provideFileSystemSuggestions(err) - case utils.ErrorTypeValidation: + case gibidiutils.ErrorTypeValidation: ef.provideValidationSuggestions(err) - case utils.ErrorTypeProcessing: + case gibidiutils.ErrorTypeProcessing: ef.provideProcessingSuggestions(err) - case utils.ErrorTypeIO: + case gibidiutils.ErrorTypeIO: ef.provideIOSuggestions(err) default: ef.provideDefaultSuggestions() @@ -85,17 +92,17 @@ func (ef *ErrorFormatter) provideSuggestions(err *utils.StructuredError) { } // provideFileSystemSuggestions provides suggestions for file system errors. -func (ef *ErrorFormatter) provideFileSystemSuggestions(err *utils.StructuredError) { +func (ef *ErrorFormatter) provideFileSystemSuggestions(err *gibidiutils.StructuredError) { filePath := err.FilePath ef.ui.PrintWarning("Suggestions:") switch err.Code { - case utils.CodeFSAccess: + case gibidiutils.CodeFSAccess: ef.suggestFileAccess(filePath) - case utils.CodeFSPathResolution: + case gibidiutils.CodeFSPathResolution: ef.suggestPathResolution(filePath) - case utils.CodeFSNotFound: + case gibidiutils.CodeFSNotFound: ef.suggestFileNotFound(filePath) default: ef.suggestFileSystemGeneral(filePath) @@ -103,91 +110,91 @@ func (ef *ErrorFormatter) provideFileSystemSuggestions(err *utils.StructuredErro } // provideValidationSuggestions provides suggestions for validation errors. -func (ef *ErrorFormatter) provideValidationSuggestions(err *utils.StructuredError) { +func (ef *ErrorFormatter) provideValidationSuggestions(err *gibidiutils.StructuredError) { ef.ui.PrintWarning("Suggestions:") switch err.Code { - case utils.CodeValidationFormat: - ef.ui.printf(" • Use a supported format: markdown, json, yaml\n") - ef.ui.printf(" • Example: -format markdown\n") - case utils.CodeValidationSize: - ef.ui.printf(" • Increase file size limit in config.yaml\n") - ef.ui.printf(" • Use smaller files or exclude large files\n") + case gibidiutils.CodeValidationFormat: + ef.ui.printf(" %s Use a supported format: markdown, json, yaml\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Example: -format markdown\n", gibidiutils.IconBullet) + case gibidiutils.CodeValidationSize: + ef.ui.printf(" %s Increase file size limit in config.yaml\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Use smaller files or exclude large files\n", gibidiutils.IconBullet) default: - ef.ui.printf(" • Check your command line arguments\n") - ef.ui.printf(" • Run with --help for usage information\n") + ef.ui.printf(" %s Check your command line arguments\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Run with --help for usage information\n", gibidiutils.IconBullet) } } // provideProcessingSuggestions provides suggestions for processing errors. -func (ef *ErrorFormatter) provideProcessingSuggestions(err *utils.StructuredError) { +func (ef *ErrorFormatter) provideProcessingSuggestions(err *gibidiutils.StructuredError) { ef.ui.PrintWarning("Suggestions:") switch err.Code { - case utils.CodeProcessingCollection: - ef.ui.printf(" • Check if the source directory exists and is readable\n") - ef.ui.printf(" • Verify directory permissions\n") - case utils.CodeProcessingFileRead: - ef.ui.printf(" • Check file permissions\n") - ef.ui.printf(" • Verify the file is not corrupted\n") + case gibidiutils.CodeProcessingCollection: + ef.ui.printf(" %s Check if the source directory exists and is readable\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Verify directory permissions\n", gibidiutils.IconBullet) + case gibidiutils.CodeProcessingFileRead: + ef.ui.printf(" %s Check file permissions\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Verify the file is not corrupted\n", gibidiutils.IconBullet) default: - ef.ui.printf(" • Try reducing concurrency: -concurrency 1\n") - ef.ui.printf(" • Check available system resources\n") + ef.ui.printf(" %s Try reducing concurrency: -concurrency 1\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Check available system resources\n", gibidiutils.IconBullet) } } // provideIOSuggestions provides suggestions for I/O errors. -func (ef *ErrorFormatter) provideIOSuggestions(err *utils.StructuredError) { +func (ef *ErrorFormatter) provideIOSuggestions(err *gibidiutils.StructuredError) { ef.ui.PrintWarning("Suggestions:") switch err.Code { - case utils.CodeIOFileCreate: - ef.ui.printf(" • Check if the destination directory exists\n") - ef.ui.printf(" • Verify write permissions for the output file\n") - ef.ui.printf(" • Ensure sufficient disk space\n") - case utils.CodeIOWrite: - ef.ui.printf(" • Check available disk space\n") - ef.ui.printf(" • Verify write permissions\n") + case gibidiutils.CodeIOFileCreate: + ef.ui.printf(" %s Check if the destination directory exists\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Verify write permissions for the output file\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Ensure sufficient disk space\n", gibidiutils.IconBullet) + case gibidiutils.CodeIOWrite: + ef.ui.printf(" %s Check available disk space\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Verify write permissions\n", gibidiutils.IconBullet) default: - ef.ui.printf(" • Check file/directory permissions\n") - ef.ui.printf(" • Verify available disk space\n") + ef.ui.printf(suggestionCheckPermissions, gibidiutils.IconBullet) + ef.ui.printf(" %s Verify available disk space\n", gibidiutils.IconBullet) } } // Helper methods for specific suggestions func (ef *ErrorFormatter) suggestFileAccess(filePath string) { - ef.ui.printf(" • Check if the path exists: %s\n", filePath) - ef.ui.printf(" • Verify read permissions\n") + ef.ui.printf(" %s Check if the path exists: %s\n", gibidiutils.IconBullet, filePath) + ef.ui.printf(" %s Verify read permissions\n", gibidiutils.IconBullet) if filePath != "" { if stat, err := os.Stat(filePath); err == nil { - ef.ui.printf(" • Path exists but may not be accessible\n") - ef.ui.printf(" • Mode: %s\n", stat.Mode()) + ef.ui.printf(" %s Path exists but may not be accessible\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Mode: %s\n", gibidiutils.IconBullet, stat.Mode()) } } } func (ef *ErrorFormatter) suggestPathResolution(filePath string) { - ef.ui.printf(" • Use an absolute path instead of relative\n") + ef.ui.printf(" %s Use an absolute path instead of relative\n", gibidiutils.IconBullet) if filePath != "" { if abs, err := filepath.Abs(filePath); err == nil { - ef.ui.printf(" • Try: %s\n", abs) + ef.ui.printf(" %s Try: %s\n", gibidiutils.IconBullet, abs) } } } func (ef *ErrorFormatter) suggestFileNotFound(filePath string) { - ef.ui.printf(" • Check if the file/directory exists: %s\n", filePath) + ef.ui.printf(" %s Check if the file/directory exists: %s\n", gibidiutils.IconBullet, filePath) if filePath != "" { dir := filepath.Dir(filePath) if entries, err := os.ReadDir(dir); err == nil { - ef.ui.printf(" • Similar files in %s:\n", dir) + ef.ui.printf(" %s Similar files in %s:\n", gibidiutils.IconBullet, dir) count := 0 for _, entry := range entries { if count >= 3 { break } if strings.Contains(entry.Name(), filepath.Base(filePath)) { - ef.ui.printf(" - %s\n", entry.Name()) + ef.ui.printf(" %s %s\n", gibidiutils.IconBullet, entry.Name()) count++ } } @@ -196,18 +203,18 @@ func (ef *ErrorFormatter) suggestFileNotFound(filePath string) { } func (ef *ErrorFormatter) suggestFileSystemGeneral(filePath string) { - ef.ui.printf(" • Check file/directory permissions\n") - ef.ui.printf(" • Verify the path is correct\n") + ef.ui.printf(suggestionCheckPermissions, gibidiutils.IconBullet) + ef.ui.printf(" %s Verify the path is correct\n", gibidiutils.IconBullet) if filePath != "" { - ef.ui.printf(" • Path: %s\n", filePath) + ef.ui.printf(" %s Path: %s\n", gibidiutils.IconBullet, filePath) } } // provideDefaultSuggestions provides general suggestions. func (ef *ErrorFormatter) provideDefaultSuggestions() { - ef.ui.printf(" • Check your command line arguments\n") - ef.ui.printf(" • Run with --help for usage information\n") - ef.ui.printf(" • Try with -concurrency 1 to reduce resource usage\n") + ef.ui.printf(" %s Check your command line arguments\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Run with --help for usage information\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Try with -concurrency 1 to reduce resource usage\n", gibidiutils.IconBullet) } // provideGenericSuggestions provides suggestions for generic errors. @@ -219,14 +226,14 @@ func (ef *ErrorFormatter) provideGenericSuggestions(err error) { // Pattern matching for common errors switch { case strings.Contains(errorMsg, "permission denied"): - ef.ui.printf(" • Check file/directory permissions\n") - ef.ui.printf(" • Try running with appropriate privileges\n") + ef.ui.printf(suggestionCheckPermissions, gibidiutils.IconBullet) + ef.ui.printf(" %s Try running with appropriate privileges\n", gibidiutils.IconBullet) case strings.Contains(errorMsg, "no such file or directory"): - ef.ui.printf(" • Verify the file/directory path is correct\n") - ef.ui.printf(" • Check if the file exists\n") + ef.ui.printf(" %s Verify the file/directory path is correct\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Check if the file exists\n", gibidiutils.IconBullet) case strings.Contains(errorMsg, "flag") && strings.Contains(errorMsg, "redefined"): - ef.ui.printf(" • This is likely a test environment issue\n") - ef.ui.printf(" • Try running the command directly instead of in tests\n") + ef.ui.printf(" %s This is likely a test environment issue\n", gibidiutils.IconBullet) + ef.ui.printf(" %s Try running the command directly instead of in tests\n", gibidiutils.IconBullet) default: ef.provideDefaultSuggestions() } @@ -234,16 +241,16 @@ func (ef *ErrorFormatter) provideGenericSuggestions(err error) { // CLI-specific error types -// CLIMissingSourceError represents a missing source directory error. -type CLIMissingSourceError struct{} +// MissingSourceError represents a missing source directory error. +type MissingSourceError struct{} -func (e CLIMissingSourceError) Error() string { +func (e MissingSourceError) Error() string { return "source directory is required" } -// NewCLIMissingSourceError creates a new CLI missing source error with suggestions. -func NewCLIMissingSourceError() error { - return &CLIMissingSourceError{} +// NewMissingSourceError creates a new CLI missing source error with suggestions. +func NewMissingSourceError() error { + return &MissingSourceError{} } // IsUserError checks if an error is a user input error that should be handled gracefully. @@ -253,16 +260,17 @@ func IsUserError(err error) bool { } // Check for specific user error types - var cliErr *CLIMissingSourceError + var cliErr *MissingSourceError if errors.As(err, &cliErr) { return true } // Check for structured errors that are user-facing - if structErr, ok := err.(*utils.StructuredError); ok { - return structErr.Type == utils.ErrorTypeValidation || - structErr.Code == utils.CodeValidationFormat || - structErr.Code == utils.CodeValidationSize + var structErr *gibidiutils.StructuredError + if errors.As(err, &structErr) { + return structErr.Type == gibidiutils.ErrorTypeValidation || + structErr.Code == gibidiutils.CodeValidationFormat || + structErr.Code == gibidiutils.CodeValidationSize } // Check error message patterns diff --git a/cli/errors_test.go b/cli/errors_test.go new file mode 100644 index 0000000..7a2b9e9 --- /dev/null +++ b/cli/errors_test.go @@ -0,0 +1,963 @@ +package cli + +import ( + "bytes" + "errors" + "strings" + "testing" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" + + "github.com/ivuorinen/gibidify/gibidiutils" +) + +func TestNewErrorFormatter(t *testing.T) { + ui := &UIManager{ + output: &bytes.Buffer{}, + } + + ef := NewErrorFormatter(ui) + + assert.NotNil(t, ef) + assert.Equal(t, ui, ef.ui) +} + +func TestFormatError(t *testing.T) { + tests := []struct { + name string + err error + expectedOutput []string + notExpected []string + }{ + { + name: "nil error", + err: nil, + expectedOutput: []string{}, + }, + { + name: "structured error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSNotFound, + testErrFileNotFound, + "/test/file.txt", + map[string]interface{}{"size": 1024}, + ), + expectedOutput: []string{ + gibidiutils.IconError + testErrorSuffix, + "FileSystem", + testErrFileNotFound, + "/test/file.txt", + "NOT_FOUND", + }, + }, + { + name: "generic error", + err: errors.New("something went wrong"), + expectedOutput: []string{gibidiutils.IconError + testErrorSuffix, "something went wrong"}, + }, + { + name: "wrapped structured error", + err: gibidiutils.WrapError( + errors.New("inner error"), + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationRequired, + "validation failed", + ), + expectedOutput: []string{ + gibidiutils.IconError + testErrorSuffix, + "validation failed", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + prev := color.NoColor + color.NoColor = true + t.Cleanup(func() { color.NoColor = prev }) + + ef := NewErrorFormatter(ui) + ef.FormatError(tt.err) + + output := buf.String() + for _, expected := range tt.expectedOutput { + assert.Contains(t, output, expected) + } + for _, notExpected := range tt.notExpected { + assert.NotContains(t, output, notExpected) + } + }) + } +} + +func TestFormatStructuredError(t *testing.T) { + tests := []struct { + name string + err *gibidiutils.StructuredError + expectedOutput []string + }{ + { + name: "filesystem error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSPermission, + testErrPermissionDenied, + "/etc/shadow", + nil, + ), + expectedOutput: []string{ + "FileSystem", + testErrPermissionDenied, + "/etc/shadow", + "PERMISSION_DENIED", + testSuggestionsHeader, + }, + }, + { + name: "validation error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + testErrInvalidFormat, + "", + map[string]interface{}{"format": "xml"}, + ), + expectedOutput: []string{ + "Validation", + testErrInvalidFormat, + "FORMAT", + testSuggestionsHeader, + }, + }, + { + name: "processing error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingFileRead, + "failed to read file", + "large.bin", + nil, + ), + expectedOutput: []string{ + "Processing", + "failed to read file", + "large.bin", + "FILE_READ", + testSuggestionsHeader, + }, + }, + { + name: "IO error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileWrite, + "disk full", + "/output/result.txt", + nil, + ), + expectedOutput: []string{ + "IO", + "disk full", + "/output/result.txt", + "FILE_WRITE", + testSuggestionsHeader, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + prev := color.NoColor + color.NoColor = true + t.Cleanup(func() { color.NoColor = prev }) + + ef := &ErrorFormatter{ui: ui} + ef.formatStructuredError(tt.err) + + output := buf.String() + for _, expected := range tt.expectedOutput { + assert.Contains(t, output, expected) + } + }) + } +} + +func TestFormatGenericError(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + prev := color.NoColor + color.NoColor = true + t.Cleanup(func() { color.NoColor = prev }) + + ef := &ErrorFormatter{ui: ui} + ef.formatGenericError(errors.New("generic error message")) + + output := buf.String() + assert.Contains(t, output, gibidiutils.IconError+testErrorSuffix) + assert.Contains(t, output, "generic error message") +} + +func TestProvideSuggestions(t *testing.T) { + tests := []struct { + name string + err *gibidiutils.StructuredError + expectedSugges []string + }{ + { + name: "filesystem permission error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSPermission, + testErrPermissionDenied, + "/root/file", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestVerifyPath, + }, + }, + { + name: "filesystem not found error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSNotFound, + testErrFileNotFound, + "/missing/file", + nil, + ), + expectedSugges: []string{ + "Check if the file/directory exists: /missing/file", + }, + }, + { + name: "validation format error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + "unsupported format", + "", + nil, + ), + expectedSugges: []string{ + testSuggestFormat, + testSuggestFormatEx, + }, + }, + { + name: "validation path error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "invalid path", + "../../etc", + nil, + ), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + }, + }, + { + name: "processing file read error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingFileRead, + "read error", + "corrupted.dat", + nil, + ), + expectedSugges: []string{ + "Check file permissions", + "Verify the file is not corrupted", + }, + }, + { + name: "IO file write error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileWrite, + "write failed", + "/output.txt", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestDiskSpace, + }, + }, + { + name: "unknown error type", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeUnknown, + "UNKNOWN", + "unknown error", + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckArgs, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + prev := color.NoColor + color.NoColor = true + t.Cleanup(func() { color.NoColor = prev }) + + ef := &ErrorFormatter{ui: ui} + ef.provideSuggestions(tt.err) + + output := buf.String() + for _, suggestion := range tt.expectedSugges { + assert.Contains(t, output, suggestion) + } + }) + } +} + +func TestProvideFileSystemSuggestions(t *testing.T) { + tests := []struct { + name string + err *gibidiutils.StructuredError + expectedSugges []string + }{ + { + name: testErrPermissionDenied, + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSPermission, + testErrPermissionDenied, + "/root/secret", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestVerifyPath, + }, + }, + { + name: "path resolution error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSPathResolution, + "path error", + "../../../etc", + nil, + ), + expectedSugges: []string{ + "Use an absolute path instead of relative", + }, + }, + { + name: testErrFileNotFound, + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + gibidiutils.CodeFSNotFound, + "not found", + "/missing.txt", + nil, + ), + expectedSugges: []string{ + "Check if the file/directory exists: /missing.txt", + }, + }, + { + name: "default filesystem error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeFileSystem, + "OTHER_FS_ERROR", + testErrOther, + "/some/path", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestVerifyPath, + "Path: /some/path", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + ef := &ErrorFormatter{ui: ui} + ef.provideFileSystemSuggestions(tt.err) + + output := buf.String() + for _, suggestion := range tt.expectedSugges { + assert.Contains(t, output, suggestion) + } + }) + } +} + +func TestProvideValidationSuggestions(t *testing.T) { + tests := []struct { + name string + err *gibidiutils.StructuredError + expectedSugges []string + }{ + { + name: "format validation", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + testErrInvalidFormat, + "", + nil, + ), + expectedSugges: []string{ + testSuggestFormat, + testSuggestFormatEx, + }, + }, + { + name: "path validation", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "invalid path", + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + }, + }, + { + name: "size validation", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationSize, + "size error", + "", + nil, + ), + expectedSugges: []string{ + "Increase file size limit in config.yaml", + "Use smaller files or exclude large files", + }, + }, + { + name: "required validation", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationRequired, + "required", + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + }, + }, + { + name: "default validation", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + "OTHER_VALIDATION", + "other", + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + ef := &ErrorFormatter{ui: ui} + ef.provideValidationSuggestions(tt.err) + + output := buf.String() + for _, suggestion := range tt.expectedSugges { + assert.Contains(t, output, suggestion) + } + }) + } +} + +func TestProvideProcessingSuggestions(t *testing.T) { + tests := []struct { + name string + err *gibidiutils.StructuredError + expectedSugges []string + }{ + { + name: "file read error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingFileRead, + "read error", + "", + nil, + ), + expectedSugges: []string{ + "Check file permissions", + "Verify the file is not corrupted", + }, + }, + { + name: "collection error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "collection error", + "", + nil, + ), + expectedSugges: []string{ + "Check if the source directory exists and is readable", + "Verify directory permissions", + }, + }, + { + name: testErrEncoding, + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingEncode, + testErrEncoding, + "", + nil, + ), + expectedSugges: []string{ + "Try reducing concurrency: -concurrency 1", + "Check available system resources", + }, + }, + { + name: "default processing", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeProcessing, + "OTHER", + testErrOther, + "", + nil, + ), + expectedSugges: []string{ + "Try reducing concurrency: -concurrency 1", + "Check available system resources", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + ef := &ErrorFormatter{ui: ui} + ef.provideProcessingSuggestions(tt.err) + + output := buf.String() + for _, suggestion := range tt.expectedSugges { + assert.Contains(t, output, suggestion) + } + }) + } +} + +func TestProvideIOSuggestions(t *testing.T) { + tests := []struct { + name string + err *gibidiutils.StructuredError + expectedSugges []string + }{ + { + name: "file create error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileCreate, + "create error", + "", + nil, + ), + expectedSugges: []string{ + "Check if the destination directory exists", + "Verify write permissions for the output file", + "Ensure sufficient disk space", + }, + }, + { + name: "file write error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileWrite, + "write error", + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestDiskSpace, + }, + }, + { + name: testErrEncoding, + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOEncoding, + testErrEncoding, + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestDiskSpace, + }, + }, + { + name: "default IO error", + err: gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + "OTHER", + testErrOther, + "", + nil, + ), + expectedSugges: []string{ + testSuggestCheckPerms, + testSuggestDiskSpace, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + ef := &ErrorFormatter{ui: ui} + ef.provideIOSuggestions(tt.err) + + output := buf.String() + for _, suggestion := range tt.expectedSugges { + assert.Contains(t, output, suggestion) + } + }) + } +} + +func TestProvideGenericSuggestions(t *testing.T) { + tests := []struct { + name string + err error + expectedSugges []string + }{ + { + name: "permission error", + err: errors.New("permission denied accessing file"), + expectedSugges: []string{ + testSuggestCheckPerms, + "Try running with appropriate privileges", + }, + }, + { + name: "not found error", + err: errors.New("no such file or directory"), + expectedSugges: []string{ + "Verify the file/directory path is correct", + "Check if the file exists", + }, + }, + { + name: "memory error", + err: errors.New("out of memory"), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + testSuggestReduceConcur, + }, + }, + { + name: "timeout error", + err: errors.New("operation timed out"), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + testSuggestReduceConcur, + }, + }, + { + name: "connection error", + err: errors.New("connection refused"), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + testSuggestReduceConcur, + }, + }, + { + name: "default error", + err: errors.New("unknown error occurred"), + expectedSugges: []string{ + testSuggestCheckArgs, + testSuggestHelp, + testSuggestReduceConcur, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + ef := &ErrorFormatter{ui: ui} + ef.provideGenericSuggestions(tt.err) + + output := buf.String() + for _, suggestion := range tt.expectedSugges { + assert.Contains(t, output, suggestion) + } + }) + } +} + +func TestMissingSourceError(t *testing.T) { + err := &MissingSourceError{} + + assert.Equal(t, "source directory is required", err.Error()) +} + +func TestNewMissingSourceErrorType(t *testing.T) { + err := NewMissingSourceError() + + assert.NotNil(t, err) + assert.Equal(t, "source directory is required", err.Error()) + + var msErr *MissingSourceError + ok := errors.As(err, &msErr) + assert.True(t, ok) + assert.NotNil(t, msErr) +} + +// Test error formatting with colors enabled +func TestFormatErrorWithColors(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: true, + output: buf, + } + prev := color.NoColor + color.NoColor = false + t.Cleanup(func() { color.NoColor = prev }) + + ef := NewErrorFormatter(ui) + err := gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + testErrInvalidFormat, + "", + nil, + ) + + ef.FormatError(err) + + output := buf.String() + // When colors are enabled, some output may go directly to stdout + // Check for suggestions that are captured in the buffer + assert.Contains(t, output, testSuggestFormat) + assert.Contains(t, output, testSuggestFormatEx) +} + +// Test wrapped error handling +func TestFormatWrappedError(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + ef := NewErrorFormatter(ui) + + innerErr := errors.New("inner error") + wrappedErr := gibidiutils.WrapError( + innerErr, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingFileRead, + "wrapper message", + ) + + ef.FormatError(wrappedErr) + + output := buf.String() + assert.Contains(t, output, "wrapper message") +} + +// Test all suggestion paths get called +func TestSuggestionPathCoverage(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + ef := &ErrorFormatter{ui: ui} + + // Test all error types + errorTypes := []gibidiutils.ErrorType{ + gibidiutils.ErrorTypeFileSystem, + gibidiutils.ErrorTypeValidation, + gibidiutils.ErrorTypeProcessing, + gibidiutils.ErrorTypeIO, + gibidiutils.ErrorTypeConfiguration, + gibidiutils.ErrorTypeUnknown, + } + + for _, errType := range errorTypes { + t.Run(errType.String(), func(t *testing.T) { + buf.Reset() + err := gibidiutils.NewStructuredError( + errType, + "TEST_CODE", + "test error", + "/test/path", + nil, + ) + ef.provideSuggestions(err) + + output := buf.String() + // Should have some suggestion output + assert.NotEmpty(t, output) + }) + } +} + +// Test suggestion helper functions with various inputs +func TestSuggestHelpers(t *testing.T) { + tests := []struct { + name string + testFunc func(*ErrorFormatter) + }{ + { + name: "suggestFileAccess", + testFunc: func(ef *ErrorFormatter) { + ef.suggestFileAccess("/root/file") + }, + }, + { + name: "suggestPathResolution", + testFunc: func(ef *ErrorFormatter) { + ef.suggestPathResolution("../../../etc") + }, + }, + { + name: "suggestFileNotFound", + testFunc: func(ef *ErrorFormatter) { + ef.suggestFileNotFound("/missing") + }, + }, + { + name: "suggestFileSystemGeneral", + testFunc: func(ef *ErrorFormatter) { + ef.suggestFileSystemGeneral("/path") + }, + }, + { + name: "provideDefaultSuggestions", + testFunc: func(ef *ErrorFormatter) { + ef.provideDefaultSuggestions() + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + ef := &ErrorFormatter{ui: ui} + + tt.testFunc(ef) + + output := buf.String() + // Each should produce some output + assert.NotEmpty(t, output) + // Should contain bullet point + assert.Contains(t, output, gibidiutils.IconBullet) + }) + } +} + +// Test edge cases in error message analysis +func TestGenericSuggestionsEdgeCases(t *testing.T) { + tests := []struct { + name string + err error + }{ + {"empty message", errors.New("")}, + {"very long message", errors.New(strings.Repeat("error ", 100))}, + {"special characters", errors.New("error!@#$%^&*()")}, + {"newlines", errors.New("error\nwith\nnewlines")}, + {"unicode", errors.New("error with 中文 characters")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + ef := &ErrorFormatter{ui: ui} + + // Should not panic + ef.provideGenericSuggestions(tt.err) + + output := buf.String() + // Should have some output + assert.NotEmpty(t, output) + }) + } +} diff --git a/cli/flags.go b/cli/flags.go index 4ca831d..6f366dc 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -5,7 +5,7 @@ import ( "runtime" "github.com/ivuorinen/gibidify/config" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // Flags holds CLI flags values. @@ -39,8 +39,10 @@ func ParseFlags() (*Flags, error) { flag.StringVar(&flags.Prefix, "prefix", "", "Text to add at the beginning of the output file") flag.StringVar(&flags.Suffix, "suffix", "", "Text to add at the end of the output file") flag.StringVar(&flags.Format, "format", "markdown", "Output format (json, markdown, yaml)") - flag.IntVar(&flags.Concurrency, "concurrency", runtime.NumCPU(), - "Number of concurrent workers (default: number of CPU cores)") + flag.IntVar( + &flags.Concurrency, "concurrency", runtime.NumCPU(), + "Number of concurrent workers (default: number of CPU cores)", + ) flag.BoolVar(&flags.NoColors, "no-colors", false, "Disable colored output") flag.BoolVar(&flags.NoProgress, "no-progress", false, "Disable progress bars") flag.BoolVar(&flags.Verbose, "verbose", false, "Enable verbose output") @@ -63,11 +65,11 @@ func ParseFlags() (*Flags, error) { // validate validates the CLI flags. func (f *Flags) validate() error { if f.SourceDir == "" { - return NewCLIMissingSourceError() + return NewMissingSourceError() } // Validate source path for security - if err := utils.ValidateSourcePath(f.SourceDir); err != nil { + if err := gibidiutils.ValidateSourcePath(f.SourceDir); err != nil { return err } @@ -77,28 +79,20 @@ func (f *Flags) validate() error { } // Validate concurrency - if err := config.ValidateConcurrency(f.Concurrency); err != nil { - return err - } - - return nil + return config.ValidateConcurrency(f.Concurrency) } // setDefaultDestination sets the default destination if not provided. func (f *Flags) setDefaultDestination() error { if f.Destination == "" { - absRoot, err := utils.GetAbsolutePath(f.SourceDir) + absRoot, err := gibidiutils.GetAbsolutePath(f.SourceDir) if err != nil { return err } - baseName := utils.GetBaseName(absRoot) + baseName := gibidiutils.GetBaseName(absRoot) f.Destination = baseName + "." + f.Format } // Validate destination path for security - if err := utils.ValidateDestinationPath(f.Destination); err != nil { - return err - } - - return nil + return gibidiutils.ValidateDestinationPath(f.Destination) } diff --git a/cli/flags_test.go b/cli/flags_test.go new file mode 100644 index 0000000..3c9fd70 --- /dev/null +++ b/cli/flags_test.go @@ -0,0 +1,366 @@ +package cli + +import ( + "errors" + "flag" + "os" + "runtime" + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestParseFlags(t *testing.T) { + // Save original command line args and restore after test + oldArgs := os.Args + oldFlagsParsed := flagsParsed + defer func() { + os.Args = oldArgs + flagsParsed = oldFlagsParsed + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + }() + + tests := []struct { + name string + args []string + expectedError string + validate func(t *testing.T, f *Flags) + setup func(t *testing.T) + }{ + { + name: "valid flags with all options", + args: []string{ + "gibidify", + testFlagSource, "", // will set to tempDir in test body + "-destination", "output.md", + "-format", "json", + testFlagConcurrency, "4", + "-prefix", "prefix", + "-suffix", "suffix", + "-no-colors", + "-no-progress", + "-verbose", + }, + validate: nil, // set in test body using closure + }, + { + name: "missing source directory", + args: []string{"gibidify"}, + expectedError: testErrSourceRequired, + }, + { + name: "invalid format", + args: []string{ + "gibidify", + testFlagSource, "", // will set to tempDir in test body + "-format", "invalid", + }, + expectedError: "unsupported output format: invalid", + }, + { + name: "invalid concurrency (zero)", + args: []string{ + "gibidify", + testFlagSource, "", // will set to tempDir in test body + testFlagConcurrency, "0", + }, + expectedError: "concurrency (0) must be at least 1", + }, + { + name: "invalid concurrency (too high)", + args: []string{ + "gibidify", + testFlagSource, "", // will set to tempDir in test body + testFlagConcurrency, "200", + }, + // Set maxConcurrency so the upper bound is enforced + expectedError: "concurrency (200) exceeds maximum (128)", + setup: func(t *testing.T) { + orig := viper.Get("maxConcurrency") + viper.Set("maxConcurrency", 128) + t.Cleanup(func() { viper.Set("maxConcurrency", orig) }) + }, + }, + { + name: "path traversal in source", + args: []string{ + "gibidify", + testFlagSource, testPathTraversalPath, + }, + expectedError: testErrPathTraversal, + }, + { + name: "default values", + args: []string{ + "gibidify", + testFlagSource, "", // will set to tempDir in test body + }, + validate: nil, // set in test body using closure + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Reset flags for each test + flagsParsed = false + globalFlags = nil + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + + // Create a local copy of args to avoid corrupting shared test data + args := append([]string{}, tt.args...) + + // Use t.TempDir for source directory if needed + tempDir := "" + for i := range args { + if i > 0 && args[i-1] == testFlagSource && args[i] == "" { + tempDir = t.TempDir() + args[i] = tempDir + } + } + os.Args = args + + // Set validate closure if needed (for tempDir) + if tt.name == "valid flags with all options" { + tt.validate = func(t *testing.T, f *Flags) { + assert.Equal(t, tempDir, f.SourceDir) + assert.Equal(t, "output.md", f.Destination) + assert.Equal(t, "json", f.Format) + assert.Equal(t, 4, f.Concurrency) + assert.Equal(t, "prefix", f.Prefix) + assert.Equal(t, "suffix", f.Suffix) + assert.True(t, f.NoColors) + assert.True(t, f.NoProgress) + assert.True(t, f.Verbose) + } + } + if tt.name == "default values" { + tt.validate = func(t *testing.T, f *Flags) { + assert.Equal(t, tempDir, f.SourceDir) + assert.Equal(t, "markdown", f.Format) + assert.Equal(t, runtime.NumCPU(), f.Concurrency) + assert.Equal(t, "", f.Prefix) + assert.Equal(t, "", f.Suffix) + assert.False(t, f.NoColors) + assert.False(t, f.NoProgress) + assert.False(t, f.Verbose) + // Destination should be set by setDefaultDestination + assert.NotEmpty(t, f.Destination) + } + } + + // Call setup if present (e.g. for maxConcurrency) + if tt.setup != nil { + tt.setup(t) + } + + flags, err := ParseFlags() + + if tt.expectedError != "" { + if assert.Error(t, err) { + assert.Contains(t, err.Error(), tt.expectedError) + } + assert.Nil(t, flags) + } else { + assert.NoError(t, err) + assert.NotNil(t, flags) + if tt.validate != nil { + tt.validate(t, flags) + } + } + }) + } +} + +func TestFlagsValidate(t *testing.T) { + tests := []struct { + name string + flags *Flags + setupFunc func(t *testing.T, f *Flags) + expectedError string + }{ + { + name: "missing source directory", + flags: &Flags{}, + expectedError: testErrSourceRequired, + }, + { + name: "invalid format", + flags: &Flags{ + Format: "invalid", + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + expectedError: "unsupported output format: invalid", + }, + { + name: "invalid concurrency", + flags: &Flags{ + Format: "markdown", + Concurrency: 0, + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + expectedError: "concurrency (0) must be at least 1", + }, + { + name: "path traversal attempt", + flags: &Flags{ + SourceDir: testPathTraversalPath, + Format: "markdown", + }, + expectedError: testErrPathTraversal, + }, + { + name: "valid flags", + flags: &Flags{ + Format: "json", + Concurrency: 4, + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc(t, tt.flags) + } + + err := tt.flags.validate() + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestSetDefaultDestination(t *testing.T) { + tests := []struct { + name string + flags *Flags + setupFunc func(t *testing.T, f *Flags) + expectedDest string + expectedError string + }{ + { + name: "default destination for directory", + flags: &Flags{ + Format: "markdown", + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + expectedDest: "", // will check suffix below + }, + { + name: "default destination for json format", + flags: &Flags{ + Format: "json", + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + expectedDest: "", // will check suffix below + }, + { + name: "provided destination unchanged", + flags: &Flags{ + Format: "markdown", + Destination: "custom-output.txt", + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + expectedDest: "custom-output.txt", + }, + { + name: "path traversal in destination", + flags: &Flags{ + Format: "markdown", + Destination: testPathTraversalPath, + }, + setupFunc: func(t *testing.T, f *Flags) { + f.SourceDir = t.TempDir() + }, + expectedError: testErrPathTraversal, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc(t, tt.flags) + } + + err := tt.flags.setDefaultDestination() + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + switch { + case tt.expectedDest != "": + assert.Equal(t, tt.expectedDest, tt.flags.Destination) + case tt.flags.Format == "json": + assert.True( + t, strings.HasSuffix(tt.flags.Destination, ".json"), + "expected %q to have suffix .json", tt.flags.Destination, + ) + case tt.flags.Format == "markdown": + assert.True( + t, strings.HasSuffix(tt.flags.Destination, ".markdown"), + "expected %q to have suffix .markdown", tt.flags.Destination, + ) + } + } + }) + } +} + +func TestFlagsSingleton(t *testing.T) { + // Save original state + oldFlagsParsed := flagsParsed + oldGlobalFlags := globalFlags + defer func() { + flagsParsed = oldFlagsParsed + globalFlags = oldGlobalFlags + }() + + // Test singleton behavior + flagsParsed = true + expectedFlags := &Flags{ + SourceDir: "/test", + Format: "json", + Concurrency: 2, + } + globalFlags = expectedFlags + + // Should return cached flags without parsing + flags, err := ParseFlags() + assert.NoError(t, err) + assert.Equal(t, expectedFlags, flags) + assert.Same(t, globalFlags, flags) +} + +func TestNewMissingSourceError(t *testing.T) { + err := NewMissingSourceError() + + assert.Error(t, err) + assert.Equal(t, testErrSourceRequired, err.Error()) + + // Check if it's the right type + var missingSourceError *MissingSourceError + ok := errors.As(err, &missingSourceError) + assert.True(t, ok) +} diff --git a/cli/processor_collection.go b/cli/processor_collection.go index cd8be10..5471440 100644 --- a/cli/processor_collection.go +++ b/cli/processor_collection.go @@ -8,14 +8,19 @@ import ( "github.com/ivuorinen/gibidify/config" "github.com/ivuorinen/gibidify/fileproc" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // collectFiles collects all files to be processed. func (p *Processor) collectFiles() ([]string, error) { files, err := fileproc.CollectFiles(p.flags.SourceDir) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "error collecting files") + return nil, gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "error collecting files", + ) } logrus.Infof("Found %d files to process", len(files)) return files, nil @@ -30,9 +35,9 @@ func (p *Processor) validateFileCollection(files []string) error { // Check file count limit maxFiles := config.GetMaxFiles() if len(files) > maxFiles { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitFiles, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitFiles, fmt.Sprintf("file count (%d) exceeds maximum limit (%d)", len(files), maxFiles), "", map[string]interface{}{ @@ -51,10 +56,14 @@ func (p *Processor) validateFileCollection(files []string) error { if fileInfo, err := os.Stat(filePath); err == nil { totalSize += fileInfo.Size() if totalSize > maxTotalSize { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitTotalSize, - fmt.Sprintf("total file size (%d bytes) would exceed maximum limit (%d bytes)", totalSize, maxTotalSize), + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitTotalSize, + fmt.Sprintf( + "total file size (%d bytes) would exceed maximum limit (%d bytes)", + totalSize, + maxTotalSize, + ), "", map[string]interface{}{ "total_size": totalSize, @@ -74,4 +83,4 @@ func (p *Processor) validateFileCollection(files []string) error { logrus.Infof("Pre-validation passed: %d files, %d MB total", len(files), totalSize/1024/1024) return nil -} \ No newline at end of file +} diff --git a/cli/processor_processing.go b/cli/processor_processing.go index 40962b0..0e9c183 100644 --- a/cli/processor_processing.go +++ b/cli/processor_processing.go @@ -6,7 +6,7 @@ import ( "sync" "github.com/ivuorinen/gibidify/fileproc" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // Process executes the main file processing workflow. @@ -16,7 +16,9 @@ func (p *Processor) Process(ctx context.Context) error { defer overallCancel() // Configure file type registry - p.configureFileTypes() + if err := p.configureFileTypes(); err != nil { + return err + } // Print startup info with colors p.ui.PrintHeader("🚀 Starting gibidify") @@ -55,7 +57,7 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error { return err } defer func() { - utils.LogError("Error closing output file", outFile.Close()) + gibidiutils.LogError("Error closing output file", outFile.Close()) }() // Initialize back-pressure and channels @@ -65,7 +67,11 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error { writerDone := make(chan struct{}) // Start writer - go fileproc.StartWriter(outFile, writeCh, writerDone, p.flags.Format, p.flags.Prefix, p.flags.Suffix) + go fileproc.StartWriter(outFile, writeCh, writerDone, fileproc.WriterConfig{ + Format: p.flags.Format, + Prefix: p.flags.Prefix, + Suffix: p.flags.Suffix, + }) // Start workers var wg sync.WaitGroup @@ -92,9 +98,13 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error { // createOutputFile creates the output file. func (p *Processor) createOutputFile() (*os.File, error) { // Destination path has been validated in CLI flags validation for path traversal attempts - outFile, err := os.Create(p.flags.Destination) // #nosec G304 - destination is validated in flags.validate() + // #nosec G304 - destination is validated in flags.validate() + outFile, err := os.Create(p.flags.Destination) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileCreate, "failed to create output file").WithFilePath(p.flags.Destination) + return nil, gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOFileCreate, + "failed to create output file", + ).WithFilePath(p.flags.Destination) } return outFile, nil -} \ No newline at end of file +} diff --git a/cli/processor_simple_test.go b/cli/processor_simple_test.go new file mode 100644 index 0000000..7a5ec44 --- /dev/null +++ b/cli/processor_simple_test.go @@ -0,0 +1,265 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ivuorinen/gibidify/fileproc" +) + +func TestProcessorSimple(t *testing.T) { + t.Run("NewProcessor", func(t *testing.T) { + flags := &Flags{ + SourceDir: "/tmp/test", + Destination: "output.md", + Format: "markdown", + Concurrency: 2, + NoColors: true, + NoProgress: true, + Verbose: false, + } + + p := NewProcessor(flags) + + assert.NotNil(t, p) + assert.Equal(t, flags, p.flags) + assert.NotNil(t, p.ui) + assert.NotNil(t, p.backpressure) + assert.NotNil(t, p.resourceMonitor) + assert.False(t, p.ui.enableColors) + assert.False(t, p.ui.enableProgress) + }) + + t.Run("ConfigureFileTypes", func(t *testing.T) { + p := &Processor{ + flags: &Flags{}, + ui: NewUIManager(), + } + + // Should not panic or error + err := p.configureFileTypes() + assert.NoError(t, err) + assert.NotNil(t, p) + }) + + t.Run("CreateOutputFile", func(t *testing.T) { + // Create temp file path + tempDir := t.TempDir() + outputPath := filepath.Join(tempDir, "output.txt") + + p := &Processor{ + flags: &Flags{ + Destination: outputPath, + }, + ui: NewUIManager(), + } + + file, err := p.createOutputFile() + assert.NoError(t, err) + assert.NotNil(t, file) + + // Clean up + err = file.Close() + require.NoError(t, err) + err = os.Remove(outputPath) + require.NoError(t, err) + }) + + t.Run("ValidateFileCollection", func(t *testing.T) { + p := &Processor{ + ui: NewUIManager(), + } + + // Empty collection should be valid (just checks limits) + err := p.validateFileCollection([]string{}) + assert.NoError(t, err) + + // Small collection should be valid + err = p.validateFileCollection([]string{ + testFilePath1, + testFilePath2, + }) + assert.NoError(t, err) + }) + + t.Run("CollectFiles_EmptyDir", func(t *testing.T) { + tempDir := t.TempDir() + + p := &Processor{ + flags: &Flags{ + SourceDir: tempDir, + }, + ui: NewUIManager(), + } + + files, err := p.collectFiles() + assert.NoError(t, err) + assert.Empty(t, files) + }) + + t.Run("CollectFiles_WithFiles", func(t *testing.T) { + tempDir := t.TempDir() + + // Create test files + require.NoError(t, os.WriteFile(filepath.Join(tempDir, "test1.go"), []byte("package main"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(tempDir, "test2.go"), []byte("package test"), 0o600)) + + // Set config so no files are ignored, and restore after test + origIgnoreDirs := viper.Get("ignoreDirectories") + origFileSizeLimit := viper.Get("fileSizeLimit") + viper.Set("ignoreDirectories", []string{}) + viper.Set("fileSizeLimit", 1024*1024*10) // 10MB + t.Cleanup(func() { + viper.Set("ignoreDirectories", origIgnoreDirs) + viper.Set("fileSizeLimit", origFileSizeLimit) + }) + + p := &Processor{ + flags: &Flags{ + SourceDir: tempDir, + }, + ui: NewUIManager(), + } + + files, err := p.collectFiles() + assert.NoError(t, err) + assert.Len(t, files, 2) + }) + + t.Run("SendFiles", func(t *testing.T) { + p := &Processor{ + backpressure: fileproc.NewBackpressureManager(), + ui: NewUIManager(), + } + + ctx := context.Background() + fileCh := make(chan string, 3) + files := []string{ + testFilePath1, + testFilePath2, + } + + var wg sync.WaitGroup + wg.Add(1) + // Send files in a goroutine since it might block + go func() { + defer wg.Done() + err := p.sendFiles(ctx, files, fileCh) + assert.NoError(t, err) + }() + + // Read all files from channel + var received []string + for i := 0; i < len(files); i++ { + file := <-fileCh + received = append(received, file) + } + + assert.Equal(t, len(files), len(received)) + + // Wait for sendFiles goroutine to finish (and close fileCh) + wg.Wait() + + // Now channel should be closed + _, ok := <-fileCh + assert.False(t, ok, "channel should be closed") + }) + + t.Run("WaitForCompletion", func(t *testing.T) { + p := &Processor{ + ui: NewUIManager(), + } + + writeCh := make(chan fileproc.WriteRequest) + writerDone := make(chan struct{}) + + // Simulate writer finishing + go func() { + <-writeCh // Wait for close + close(writerDone) + }() + + var wg sync.WaitGroup + // Start and finish immediately + wg.Add(1) + wg.Done() + + // Should complete without hanging + p.waitForCompletion(&wg, writeCh, writerDone) + assert.NotNil(t, p) + }) + + t.Run("LogFinalStats", func(t *testing.T) { + p := &Processor{ + flags: &Flags{ + Verbose: true, + }, + ui: NewUIManager(), + resourceMonitor: fileproc.NewResourceMonitor(), + backpressure: fileproc.NewBackpressureManager(), + } + + // Should not panic + p.logFinalStats() + assert.NotNil(t, p) + }) +} + +// Test error handling scenarios +func TestProcessorErrors(t *testing.T) { + t.Run("CreateOutputFile_InvalidPath", func(t *testing.T) { + p := &Processor{ + flags: &Flags{ + Destination: "/root/cannot-write-here.txt", + }, + ui: NewUIManager(), + } + + file, err := p.createOutputFile() + assert.Error(t, err) + assert.Nil(t, file) + }) + + t.Run("CollectFiles_NonExistentDir", func(t *testing.T) { + p := &Processor{ + flags: &Flags{ + SourceDir: "/non/existent/path", + }, + ui: NewUIManager(), + } + + files, err := p.collectFiles() + assert.Error(t, err) + assert.Nil(t, files) + }) + + t.Run("SendFiles_WithCancellation", func(t *testing.T) { + p := &Processor{ + backpressure: fileproc.NewBackpressureManager(), + ui: NewUIManager(), + } + + ctx, cancel := context.WithCancel(context.Background()) + fileCh := make(chan string) // Unbuffered to force blocking + + files := []string{ + testFilePath1, + testFilePath2, + "/test/file3.go", + } + + // Cancel immediately + cancel() + + err := p.sendFiles(ctx, files, fileCh) + assert.Error(t, err) + assert.Equal(t, context.Canceled, err) + }) +} diff --git a/cli/processor_stats.go b/cli/processor_stats.go index 6ecd856..c55fbd7 100644 --- a/cli/processor_stats.go +++ b/cli/processor_stats.go @@ -11,8 +11,12 @@ func (p *Processor) logFinalStats() { // Log back-pressure stats backpressureStats := p.backpressure.GetStats() if backpressureStats.Enabled { - logrus.Infof("Back-pressure stats: processed=%d files, memory=%dMB/%dMB", - backpressureStats.FilesProcessed, backpressureStats.CurrentMemoryUsage/1024/1024, backpressureStats.MaxMemoryUsage/1024/1024) + logrus.Infof( + "Back-pressure stats: processed=%d files, memory=%dMB/%dMB", + backpressureStats.FilesProcessed, + backpressureStats.CurrentMemoryUsage/1024/1024, + backpressureStats.MaxMemoryUsage/1024/1024, + ) } // Log resource monitoring stats @@ -37,4 +41,4 @@ func (p *Processor) logFinalStats() { // Clean up resource monitor p.resourceMonitor.Close() -} \ No newline at end of file +} diff --git a/cli/processor_types.go b/cli/processor_types.go index e5d37e2..1675a66 100644 --- a/cli/processor_types.go +++ b/cli/processor_types.go @@ -30,15 +30,18 @@ func NewProcessor(flags *Flags) *Processor { } // configureFileTypes configures the file type registry. -func (p *Processor) configureFileTypes() { +func (p *Processor) configureFileTypes() error { if config.GetFileTypesEnabled() { - fileproc.ConfigureFromSettings( - config.GetCustomImageExtensions(), - config.GetCustomBinaryExtensions(), - config.GetCustomLanguages(), - config.GetDisabledImageExtensions(), - config.GetDisabledBinaryExtensions(), - config.GetDisabledLanguageExtensions(), - ) + if err := fileproc.ConfigureFromSettings(fileproc.RegistryConfig{ + CustomImages: config.GetCustomImageExtensions(), + CustomBinary: config.GetCustomBinaryExtensions(), + CustomLanguages: config.GetCustomLanguages(), + DisabledImages: config.GetDisabledImageExtensions(), + DisabledBinary: config.GetDisabledBinaryExtensions(), + DisabledLanguages: config.GetDisabledLanguageExtensions(), + }); err != nil { + return err + } } -} \ No newline at end of file + return nil +} diff --git a/cli/processor_workers.go b/cli/processor_workers.go index ebfac43..df67cbd 100644 --- a/cli/processor_workers.go +++ b/cli/processor_workers.go @@ -7,11 +7,16 @@ import ( "github.com/sirupsen/logrus" "github.com/ivuorinen/gibidify/fileproc" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // startWorkers starts the worker goroutines. -func (p *Processor) startWorkers(ctx context.Context, wg *sync.WaitGroup, fileCh chan string, writeCh chan fileproc.WriteRequest) { +func (p *Processor) startWorkers( + ctx context.Context, + wg *sync.WaitGroup, + fileCh chan string, + writeCh chan fileproc.WriteRequest, +) { for range p.flags.Concurrency { wg.Add(1) go p.worker(ctx, wg, fileCh, writeCh) @@ -19,7 +24,12 @@ func (p *Processor) startWorkers(ctx context.Context, wg *sync.WaitGroup, fileCh } // worker is the worker goroutine function. -func (p *Processor) worker(ctx context.Context, wg *sync.WaitGroup, fileCh chan string, writeCh chan fileproc.WriteRequest) { +func (p *Processor) worker( + ctx context.Context, + wg *sync.WaitGroup, + fileCh chan string, + writeCh chan fileproc.WriteRequest, +) { defer wg.Done() for { select { @@ -42,9 +52,9 @@ func (p *Processor) processFile(ctx context.Context, filePath string, writeCh ch return } - absRoot, err := utils.GetAbsolutePath(p.flags.SourceDir) + absRoot, err := gibidiutils.GetAbsolutePath(p.flags.SourceDir) if err != nil { - utils.LogError("Failed to get absolute path", err) + gibidiutils.LogError("Failed to get absolute path", err) return } @@ -78,8 +88,12 @@ func (p *Processor) sendFiles(ctx context.Context, files []string, fileCh chan s } // waitForCompletion waits for all workers to complete. -func (p *Processor) waitForCompletion(wg *sync.WaitGroup, writeCh chan fileproc.WriteRequest, writerDone chan struct{}) { +func (p *Processor) waitForCompletion( + wg *sync.WaitGroup, + writeCh chan fileproc.WriteRequest, + writerDone chan struct{}, +) { wg.Wait() close(writeCh) <-writerDone -} \ No newline at end of file +} diff --git a/cli/terminal_test_helpers.go b/cli/terminal_test_helpers.go new file mode 100644 index 0000000..62c9d11 --- /dev/null +++ b/cli/terminal_test_helpers.go @@ -0,0 +1,68 @@ +package cli + +import "testing" + +// terminalEnvSetup defines environment variables for terminal detection tests. +type terminalEnvSetup struct { + Term string + CI string + GitHubActions string + NoColor string + ForceColor string +} + +// apply sets up the environment variables using t.Setenv. +func (e terminalEnvSetup) apply(t *testing.T) { + t.Helper() + + // Always set all environment variables to ensure isolation + // Empty string explicitly unsets the variable in the test environment + t.Setenv("TERM", e.Term) + t.Setenv("CI", e.CI) + t.Setenv("GITHUB_ACTIONS", e.GitHubActions) + t.Setenv("NO_COLOR", e.NoColor) + t.Setenv("FORCE_COLOR", e.ForceColor) +} + +// Common terminal environment setups for reuse across tests. +var ( + envDefaultTerminal = terminalEnvSetup{ + Term: "xterm-256color", + CI: "", + NoColor: "", + ForceColor: "", + } + + envDumbTerminal = terminalEnvSetup{ + Term: "dumb", + } + + envCIWithoutGitHub = terminalEnvSetup{ + Term: "xterm", + CI: "true", + GitHubActions: "", + } + + envGitHubActions = terminalEnvSetup{ + Term: "xterm", + CI: "true", + GitHubActions: "true", + NoColor: "", + } + + envNoColor = terminalEnvSetup{ + Term: "xterm-256color", + CI: "", + NoColor: "1", + ForceColor: "", + } + + envForceColor = terminalEnvSetup{ + Term: "dumb", + ForceColor: "1", + } + + envEmptyTerm = terminalEnvSetup{ + Term: "", + } +) diff --git a/cli/test_constants.go b/cli/test_constants.go new file mode 100644 index 0000000..74dd3ee --- /dev/null +++ b/cli/test_constants.go @@ -0,0 +1,42 @@ +package cli + +// Test constants to avoid duplication in test files. +// These constants are used across multiple test files in the cli package. +const ( + // Error messages + testErrFileNotFound = "file not found" + testErrPermissionDenied = "permission denied" + testErrInvalidFormat = "invalid format" + testErrOther = "other error" + testErrEncoding = "encoding error" + testErrSourceRequired = "source directory is required" + testErrPathTraversal = "path traversal attempt detected" + testPathTraversalPath = "../../../etc/passwd" + + // Suggestion messages + testSuggestionsHeader = "Suggestions:" + testSuggestCheckPerms = "Check file/directory permissions" + testSuggestVerifyPath = "Verify the path is correct" + testSuggestFormat = "Use a supported format: markdown, json, yaml" + testSuggestFormatEx = "Example: -format markdown" + testSuggestCheckArgs = "Check your command line arguments" + testSuggestHelp = "Run with --help for usage information" + testSuggestDiskSpace = "Verify available disk space" + testSuggestReduceConcur = "Try with -concurrency 1 to reduce resource usage" + + // UI test strings + testWithColors = "with colors" + testWithoutColors = "without colors" + testProcessingMsg = "Processing files" + + // Flag names + testFlagSource = "-source" + testFlagConcurrency = "-concurrency" + + // Test file paths + testFilePath1 = "/test/file1.go" + testFilePath2 = "/test/file2.go" + + // Output markers + testErrorSuffix = " Error" +) diff --git a/cli/ui.go b/cli/ui.go index 1c7bc27..45a971e 100644 --- a/cli/ui.go +++ b/cli/ui.go @@ -8,6 +8,8 @@ import ( "github.com/fatih/color" "github.com/schollz/progressbar/v3" + + "github.com/ivuorinen/gibidify/gibidiutils" ) // UIManager handles CLI user interface elements. @@ -44,23 +46,40 @@ func (ui *UIManager) StartProgress(total int, description string) { return } - ui.progressBar = progressbar.NewOptions(total, - progressbar.OptionSetWriter(ui.output), - progressbar.OptionSetDescription(description), - progressbar.OptionSetTheme(progressbar.Theme{ + // Set progress bar theme based on color support + var theme progressbar.Theme + if ui.enableColors { + theme = progressbar.Theme{ Saucer: color.GreenString("█"), SaucerHead: color.GreenString("█"), SaucerPadding: " ", BarStart: "[", BarEnd: "]", - }), + } + } else { + theme = progressbar.Theme{ + Saucer: "█", + SaucerHead: "█", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + } + } + + ui.progressBar = progressbar.NewOptions( + total, + progressbar.OptionSetWriter(ui.output), + progressbar.OptionSetDescription(description), + progressbar.OptionSetTheme(theme), progressbar.OptionShowCount(), progressbar.OptionShowIts(), progressbar.OptionSetWidth(40), progressbar.OptionThrottle(100*time.Millisecond), - progressbar.OptionOnCompletion(func() { - _, _ = fmt.Fprint(ui.output, "\n") - }), + progressbar.OptionOnCompletion( + func() { + _, _ = fmt.Fprint(ui.output, "\n") + }, + ), progressbar.OptionSetRenderBlankState(true), ) } @@ -80,40 +99,44 @@ func (ui *UIManager) FinishProgress() { } } -// PrintSuccess prints a success message in green. +// writeMessage writes a formatted message with optional colorization. +// It handles color enablement, formatting, writing to output, and error logging. +func (ui *UIManager) writeMessage( + icon, methodName, format string, + colorFunc func(string, ...interface{}) string, + args ...interface{}, +) { + msg := icon + " " + format + var output string + if ui.enableColors && colorFunc != nil { + output = colorFunc(msg, args...) + } else { + output = fmt.Sprintf(msg, args...) + } + + if _, err := fmt.Fprintf(ui.output, "%s\n", output); err != nil { + gibidiutils.LogError(fmt.Sprintf("UIManager.%s: failed to write to output", methodName), err) + } +} + +// PrintSuccess prints a success message in green (to ui.output if set). func (ui *UIManager) PrintSuccess(format string, args ...interface{}) { - if ui.enableColors { - color.Green("✓ "+format, args...) - } else { - ui.printf("✓ "+format+"\n", args...) - } + ui.writeMessage(gibidiutils.IconSuccess, "PrintSuccess", format, color.GreenString, args...) } -// PrintError prints an error message in red. +// PrintError prints an error message in red (to ui.output if set). func (ui *UIManager) PrintError(format string, args ...interface{}) { - if ui.enableColors { - color.Red("✗ "+format, args...) - } else { - ui.printf("✗ "+format+"\n", args...) - } + ui.writeMessage(gibidiutils.IconError, "PrintError", format, color.RedString, args...) } -// PrintWarning prints a warning message in yellow. +// PrintWarning prints a warning message in yellow (to ui.output if set). func (ui *UIManager) PrintWarning(format string, args ...interface{}) { - if ui.enableColors { - color.Yellow("⚠ "+format, args...) - } else { - ui.printf("⚠ "+format+"\n", args...) - } + ui.writeMessage(gibidiutils.IconWarning, "PrintWarning", format, color.YellowString, args...) } -// PrintInfo prints an info message in blue. +// PrintInfo prints an info message in blue (to ui.output if set). func (ui *UIManager) PrintInfo(format string, args ...interface{}) { - if ui.enableColors { - color.Blue("ℹ "+format, args...) - } else { - ui.printf("ℹ "+format+"\n", args...) - } + ui.writeMessage(gibidiutils.IconInfo, "PrintInfo", format, color.BlueString, args...) } // PrintHeader prints a header message in bold. @@ -127,6 +150,11 @@ func (ui *UIManager) PrintHeader(format string, args ...interface{}) { // isColorTerminal checks if the terminal supports colors. func isColorTerminal() bool { + // Check if FORCE_COLOR is set + if os.Getenv("FORCE_COLOR") != "" { + return true + } + // Check common environment variables term := os.Getenv("TERM") if term == "" || term == "dumb" { @@ -148,13 +176,7 @@ func isColorTerminal() bool { return false } - // Check if FORCE_COLOR is set - if os.Getenv("FORCE_COLOR") != "" { - return true - } - - // Default to true for interactive terminals - return isInteractiveTerminal() + return true } // isInteractiveTerminal checks if we're running in an interactive terminal. diff --git a/cli/ui_manager_test.go b/cli/ui_manager_test.go new file mode 100644 index 0000000..5b26487 --- /dev/null +++ b/cli/ui_manager_test.go @@ -0,0 +1,109 @@ +package cli + +import ( + "bytes" + "os" + "testing" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" +) + +func TestNewUIManager(t *testing.T) { + tests := []struct { + name string + env terminalEnvSetup + expectedColors bool + expectedProgress bool + }{ + { + name: "default terminal", + env: envDefaultTerminal, + expectedColors: true, + expectedProgress: false, // Not a tty in test environment + }, + { + name: "dumb terminal", + env: envDumbTerminal, + expectedColors: false, + expectedProgress: false, + }, + { + name: "CI environment without GitHub Actions", + env: envCIWithoutGitHub, + expectedColors: false, + expectedProgress: false, + }, + { + name: "GitHub Actions CI", + env: envGitHubActions, + expectedColors: true, + expectedProgress: false, + }, + { + name: "NO_COLOR set", + env: envNoColor, + expectedColors: false, + expectedProgress: false, + }, + { + name: "FORCE_COLOR set", + env: envForceColor, + expectedColors: true, + expectedProgress: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.env.apply(t) + + ui := NewUIManager() + assert.NotNil(t, ui) + assert.NotNil(t, ui.output) + assert.Equal(t, tt.expectedColors, ui.enableColors, "color state mismatch") + assert.Equal(t, tt.expectedProgress, ui.enableProgress, "progress state mismatch") + }) + } +} + +func TestSetColorOutput(t *testing.T) { + // Capture original color.NoColor state and restore after test + orig := color.NoColor + defer func() { color.NoColor = orig }() + + ui := &UIManager{output: os.Stderr} + + // Test enabling colors + ui.SetColorOutput(true) + assert.False(t, color.NoColor) + assert.True(t, ui.enableColors) + + // Test disabling colors + ui.SetColorOutput(false) + assert.True(t, color.NoColor) + assert.False(t, ui.enableColors) +} + +func TestSetProgressOutput(t *testing.T) { + ui := &UIManager{output: os.Stderr} + + // Test enabling progress + ui.SetProgressOutput(true) + assert.True(t, ui.enableProgress) + + // Test disabling progress + ui.SetProgressOutput(false) + assert.False(t, ui.enableProgress) +} + +func TestPrintf(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + output: buf, + } + + ui.printf("Test %s %d", "output", 123) + + assert.Equal(t, "Test output 123", buf.String()) +} diff --git a/cli/ui_print_test.go b/cli/ui_print_test.go new file mode 100644 index 0000000..f21c4fa --- /dev/null +++ b/cli/ui_print_test.go @@ -0,0 +1,245 @@ +package cli + +import ( + "bytes" + "strings" + "testing" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" + + "github.com/ivuorinen/gibidify/gibidiutils" +) + +func TestPrintSuccess(t *testing.T) { + tests := []struct { + name string + enableColors bool + format string + args []interface{} + expectSymbol string + }{ + { + name: testWithColors, + enableColors: true, + format: "Operation %s", + args: []interface{}{"completed"}, + expectSymbol: gibidiutils.IconSuccess, + }, + { + name: testWithoutColors, + enableColors: false, + format: "Operation %s", + args: []interface{}{"completed"}, + expectSymbol: gibidiutils.IconSuccess, + }, + { + name: "no arguments", + enableColors: true, + format: "Success", + args: nil, + expectSymbol: gibidiutils.IconSuccess, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: tt.enableColors, + output: buf, + } + prev := color.NoColor + color.NoColor = !tt.enableColors + defer func() { color.NoColor = prev }() + + ui.PrintSuccess(tt.format, tt.args...) + + output := buf.String() + assert.Contains(t, output, tt.expectSymbol) + if len(tt.args) > 0 { + assert.Contains(t, output, "completed") + } + }, + ) + } +} + +func TestPrintError(t *testing.T) { + tests := []struct { + name string + enableColors bool + format string + args []interface{} + expectSymbol string + }{ + { + name: testWithColors, + enableColors: true, + format: "Failed to %s", + args: []interface{}{"process"}, + expectSymbol: gibidiutils.IconError, + }, + { + name: testWithoutColors, + enableColors: false, + format: "Failed to %s", + args: []interface{}{"process"}, + expectSymbol: gibidiutils.IconError, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: tt.enableColors, + output: buf, + } + prev := color.NoColor + color.NoColor = !tt.enableColors + defer func() { color.NoColor = prev }() + + ui.PrintError(tt.format, tt.args...) + + output := buf.String() + assert.Contains(t, output, tt.expectSymbol) + if len(tt.args) > 0 { + assert.Contains(t, output, "process") + } + }, + ) + } +} + +func TestPrintWarning(t *testing.T) { + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: true, + output: buf, + } + + ui.PrintWarning("This is a %s", "warning") + + output := buf.String() + assert.Contains(t, output, gibidiutils.IconWarning) +} + +func TestPrintInfo(t *testing.T) { + // Capture original color.NoColor state and restore after test + orig := color.NoColor + defer func() { color.NoColor = orig }() + + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: true, + output: buf, + } + + color.NoColor = false + + ui.PrintInfo("Information: %d items", 42) + + output := buf.String() + assert.Contains(t, output, gibidiutils.IconInfo) + assert.Contains(t, output, "42") +} + +func TestPrintHeader(t *testing.T) { + tests := []struct { + name string + enableColors bool + format string + args []interface{} + }{ + { + name: testWithColors, + enableColors: true, + format: "Header %s", + args: []interface{}{"Title"}, + }, + { + name: testWithoutColors, + enableColors: false, + format: "Header %s", + args: []interface{}{"Title"}, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + // Capture original color.NoColor state and restore after test + orig := color.NoColor + defer func() { color.NoColor = orig }() + + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: tt.enableColors, + output: buf, + } + color.NoColor = !tt.enableColors + + ui.PrintHeader(tt.format, tt.args...) + + output := buf.String() + assert.Contains(t, output, "Title") + }, + ) + } +} + +// Test that all print methods handle newlines correctly +func TestPrintMethodsNewlines(t *testing.T) { + tests := []struct { + name string + method func(*UIManager, string, ...interface{}) + symbol string + }{ + { + name: "PrintSuccess", + method: (*UIManager).PrintSuccess, + symbol: gibidiutils.IconSuccess, + }, + { + name: "PrintError", + method: (*UIManager).PrintError, + symbol: gibidiutils.IconError, + }, + { + name: "PrintWarning", + method: (*UIManager).PrintWarning, + symbol: gibidiutils.IconWarning, + }, + { + name: "PrintInfo", + method: (*UIManager).PrintInfo, + symbol: gibidiutils.IconInfo, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + // Disable colors for consistent testing + oldNoColor := color.NoColor + color.NoColor = true + defer func() { color.NoColor = oldNoColor }() + + buf := &bytes.Buffer{} + ui := &UIManager{ + enableColors: false, + output: buf, + } + + tt.method(ui, "Test message") + + output := buf.String() + assert.True(t, strings.HasSuffix(output, "\n")) + assert.Contains(t, output, tt.symbol) + }, + ) + } +} diff --git a/cli/ui_progress_test.go b/cli/ui_progress_test.go new file mode 100644 index 0000000..e9a8d6f --- /dev/null +++ b/cli/ui_progress_test.go @@ -0,0 +1,147 @@ +package cli + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStartProgress(t *testing.T) { + tests := []struct { + name string + total int + description string + enabled bool + expectBar bool + }{ + { + name: "progress enabled with valid total", + total: 100, + description: testProcessingMsg, + enabled: true, + expectBar: true, + }, + { + name: "progress disabled", + total: 100, + description: testProcessingMsg, + enabled: false, + expectBar: false, + }, + { + name: "zero total", + total: 0, + description: testProcessingMsg, + enabled: true, + expectBar: false, + }, + { + name: "negative total", + total: -5, + description: testProcessingMsg, + enabled: true, + expectBar: false, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + ui := &UIManager{ + enableProgress: tt.enabled, + output: &bytes.Buffer{}, + } + + ui.StartProgress(tt.total, tt.description) + + if tt.expectBar { + assert.NotNil(t, ui.progressBar) + } else { + assert.Nil(t, ui.progressBar) + } + }, + ) + } +} + +func TestUpdateProgress(t *testing.T) { + tests := []struct { + name string + setupBar bool + enabledProg bool + expectUpdate bool + }{ + { + name: "with progress bar", + setupBar: true, + enabledProg: true, + expectUpdate: true, + }, + { + name: "without progress bar", + setupBar: false, + enabledProg: false, + expectUpdate: false, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(_ *testing.T) { + ui := &UIManager{ + enableProgress: tt.enabledProg, + output: &bytes.Buffer{}, + } + + if tt.setupBar { + ui.StartProgress(10, "Test") + } + + // Should not panic + ui.UpdateProgress(1) + + // Multiple updates should not panic + ui.UpdateProgress(2) + ui.UpdateProgress(3) + }, + ) + } +} + +func TestFinishProgress(t *testing.T) { + tests := []struct { + name string + setupBar bool + }{ + { + name: "with progress bar", + setupBar: true, + }, + { + name: "without progress bar", + setupBar: false, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + ui := &UIManager{ + enableProgress: true, + output: &bytes.Buffer{}, + } + + if tt.setupBar { + ui.StartProgress(10, "Test") + } + + // Should not panic + ui.FinishProgress() + + // Bar should be cleared + assert.Nil(t, ui.progressBar) + }, + ) + } +} diff --git a/cli/ui_terminal_test.go b/cli/ui_terminal_test.go new file mode 100644 index 0000000..924f8e4 --- /dev/null +++ b/cli/ui_terminal_test.go @@ -0,0 +1,62 @@ +package cli + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsColorTerminal(t *testing.T) { + tests := []struct { + name string + env terminalEnvSetup + expected bool + }{ + { + name: "dumb terminal", + env: envDumbTerminal, + expected: false, + }, + { + name: "empty TERM", + env: envEmptyTerm, + expected: false, + }, + { + name: "CI without GitHub Actions", + env: envCIWithoutGitHub, + expected: false, + }, + { + name: "GitHub Actions", + env: envGitHubActions, + expected: true, + }, + { + name: "NO_COLOR set", + env: envNoColor, + expected: false, + }, + { + name: "FORCE_COLOR set", + env: envForceColor, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.env.apply(t) + + result := isColorTerminal() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestIsInteractiveTerminal(t *testing.T) { + // This function checks if stderr is a terminal + // In test environment, it will typically return false + result := isInteractiveTerminal() + assert.False(t, result) +} diff --git a/cmd/benchmark/main.go b/cmd/benchmark/main.go index e5be0ea..63ec906 100644 --- a/cmd/benchmark/main.go +++ b/cmd/benchmark/main.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/ivuorinen/gibidify/benchmark" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) var ( @@ -26,7 +26,7 @@ func main() { flag.Parse() if err := runBenchmarks(); err != nil { - fmt.Fprintf(os.Stderr, "Benchmark failed: %v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "Benchmark failed: %v\n", err) os.Exit(1) } } @@ -50,7 +50,10 @@ func runBenchmarks() error { case "format": return runFormatBenchmark() default: - return utils.NewValidationError(utils.CodeValidationFormat, "invalid benchmark type: "+*benchmarkType) + return gibidiutils.NewValidationError( + gibidiutils.CodeValidationFormat, + "invalid benchmark type: "+*benchmarkType, + ) } } @@ -58,9 +61,14 @@ func runCollectionBenchmark() error { fmt.Println("Running file collection benchmark...") result, err := benchmark.FileCollectionBenchmark(*sourceDir, *numFiles) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file collection benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "file collection benchmark failed", + ) } - benchmark.PrintBenchmarkResult(result) + benchmark.PrintResult(result) return nil } @@ -68,24 +76,39 @@ func runProcessingBenchmark() error { fmt.Printf("Running file processing benchmark (format: %s, concurrency: %d)...\n", *format, *concurrency) result, err := benchmark.FileProcessingBenchmark(*sourceDir, *format, *concurrency) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file processing benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "file processing benchmark failed", + ) } - benchmark.PrintBenchmarkResult(result) + benchmark.PrintResult(result) return nil } func runConcurrencyBenchmark() error { concurrencyLevels, err := parseConcurrencyList(*concurrencyList) if err != nil { - return utils.WrapError(err, utils.ErrorTypeValidation, utils.CodeValidationFormat, "invalid concurrency list") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + "invalid concurrency list", + ) } fmt.Printf("Running concurrency benchmark (format: %s, levels: %v)...\n", *format, concurrencyLevels) suite, err := benchmark.ConcurrencyBenchmark(*sourceDir, *format, concurrencyLevels) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "concurrency benchmark failed", + ) } - benchmark.PrintBenchmarkSuite(suite) + benchmark.PrintSuite(suite) return nil } @@ -94,9 +117,14 @@ func runFormatBenchmark() error { fmt.Printf("Running format benchmark (formats: %v)...\n", formats) suite, err := benchmark.FormatBenchmark(*sourceDir, formats) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeProcessing, + gibidiutils.CodeProcessingCollection, + "format benchmark failed", + ) } - benchmark.PrintBenchmarkSuite(suite) + benchmark.PrintSuite(suite) return nil } @@ -115,16 +143,28 @@ func parseConcurrencyList(list string) ([]int, error) { part = strings.TrimSpace(part) var level int if _, err := fmt.Sscanf(part, "%d", &level); err != nil { - return nil, utils.WrapErrorf(err, utils.ErrorTypeValidation, utils.CodeValidationFormat, "invalid concurrency level: %s", part) + return nil, gibidiutils.WrapErrorf( + err, + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + "invalid concurrency level: %s", + part, + ) } if level <= 0 { - return nil, utils.NewValidationError(utils.CodeValidationFormat, "concurrency level must be positive: "+part) + return nil, gibidiutils.NewValidationError( + gibidiutils.CodeValidationFormat, + "concurrency level must be positive: "+part, + ) } levels = append(levels, level) } if len(levels) == 0 { - return nil, utils.NewValidationError(utils.CodeValidationFormat, "no valid concurrency levels found") + return nil, gibidiutils.NewValidationError( + gibidiutils.CodeValidationFormat, + "no valid concurrency levels found", + ) } return levels, nil diff --git a/config.yaml.example b/config.yaml.example index b2aca82..8641ee8 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -38,24 +38,24 @@ backpressure: # Resource limits for DoS protection and security resourceLimits: enabled: true - + # File processing limits maxFiles: 10000 # Maximum number of files to process maxTotalSize: 1073741824 # Maximum total size (1GB) - + # Timeout limits (in seconds) fileProcessingTimeoutSec: 30 # Timeout for individual file processing overallTimeoutSec: 3600 # Overall processing timeout (1 hour) - + # Concurrency limits maxConcurrentReads: 10 # Maximum concurrent file reading operations - + # Rate limiting (0 = disabled) rateLimitFilesPerSec: 0 # Files per second rate limit - + # Memory limits hardMemoryLimitMB: 512 # Hard memory limit (512MB) - + # Safety features enableGracefulDegradation: true # Enable graceful degradation on resource pressure enableResourceMonitoring: true # Enable detailed resource monitoring @@ -76,4 +76,4 @@ resourceLimits: # filePatterns: # - "*.go" # - "*.py" -# - "*.js" \ No newline at end of file +# - "*.js" diff --git a/config/constants.go b/config/constants.go index 8f54bbe..1a2ba75 100644 --- a/config/constants.go +++ b/config/constants.go @@ -58,4 +58,4 @@ const ( MinHardMemoryLimitMB = 64 // MaxHardMemoryLimitMB is the maximum hard memory limit (8192MB = 8GB). MaxHardMemoryLimitMB = 8192 -) \ No newline at end of file +) diff --git a/config/getters.go b/config/getters.go index 4bcc1b4..b178144 100644 --- a/config/getters.go +++ b/config/getters.go @@ -154,4 +154,4 @@ func GetEnableGracefulDegradation() bool { // GetEnableResourceMonitoring returns whether resource monitoring is enabled. func GetEnableResourceMonitoring() bool { return viper.GetBool("resourceLimits.enableResourceMonitoring") -} \ No newline at end of file +} diff --git a/config/loader.go b/config/loader.go index c7490b5..18f6c68 100644 --- a/config/loader.go +++ b/config/loader.go @@ -1,13 +1,15 @@ package config import ( + "flag" "os" "path/filepath" + "sync/atomic" "github.com/sirupsen/logrus" "github.com/spf13/viper" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // LoadConfig reads configuration from a YAML file. @@ -15,13 +17,18 @@ import ( // 1. $XDG_CONFIG_HOME/gibidify/config.yaml // 2. $HOME/.config/gibidify/config.yaml // 3. The current directory as fallback. +// +// Note: LoadConfig relies on isRunningTest() which requires the testing package +// to have registered its flags (e.g., via flag.Parse() or during test initialization). +// If called too early (e.g., from init() or before TestMain), test detection may not work reliably. +// For explicit control, use SetRunningInTest() before calling LoadConfig. func LoadConfig() { viper.SetConfigName("config") viper.SetConfigType("yaml") if xdgConfig := os.Getenv("XDG_CONFIG_HOME"); xdgConfig != "" { // Validate XDG_CONFIG_HOME for path traversal attempts - if err := utils.ValidateConfigPath(xdgConfig); err != nil { + if err := gibidiutils.ValidateConfigPath(xdgConfig); err != nil { logrus.Warnf("Invalid XDG_CONFIG_HOME path, using default config: %v", err) } else { configPath := filepath.Join(xdgConfig, "gibidify") @@ -37,7 +44,14 @@ func LoadConfig() { } if err := viper.ReadInConfig(); err != nil { - logrus.Infof("Config file not found, using default values: %v", err) + // Suppress this info-level log when running tests. + // Prefer an explicit test flag (SetRunningInTest) but fall back to runtime detection. + if runningInTest.Load() || isRunningTest() { + // Keep a debug-level record so tests that enable debug can still see it. + logrus.Debugf("Config file not found (tests): %v", err) + } else { + logrus.Infof("Config file not found, using default values: %v", err) + } setDefaultConfig() } else { logrus.Infof("Using config file: %s", viper.ConfigFileUsed()) @@ -87,4 +101,31 @@ func setDefaultConfig() { viper.SetDefault("resourceLimits.hardMemoryLimitMB", DefaultHardMemoryLimitMB) viper.SetDefault("resourceLimits.enableGracefulDegradation", true) viper.SetDefault("resourceLimits.enableResourceMonitoring", true) -} \ No newline at end of file +} + +var runningInTest atomic.Bool + +// SetRunningInTest allows tests to explicitly indicate they are running under `go test`. +// Call this from TestMain in tests to suppress noisy info logs while still allowing +// debug-level output for tests that enable it. +func SetRunningInTest(b bool) { + runningInTest.Store(b) +} + +// isRunningTest attempts to detect if the binary is running under `go test`. +// Prefer checking for standard test flags registered by the testing package. +// This is reliable when `go test` initializes the flag set. +// +// IMPORTANT: This function relies on flag.Lookup which returns nil if the testing +// package hasn't registered test flags yet. Callers must invoke this after flag +// parsing (or test flag registration) has occurred. If invoked too early (e.g., +// from init() or early in TestMain before flags are parsed), detection will fail. +// For explicit control, use SetRunningInTest() instead. +func isRunningTest() bool { + // Look for the well-known test flags created by the testing package. + // If any are present in the flag registry, we're running under `go test`. + if flag.Lookup("test.v") != nil || flag.Lookup("test.run") != nil || flag.Lookup("test.bench") != nil { + return true + } + return false +} diff --git a/config/loader_test.go b/config/loader_test.go index d1c5295..6290f02 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -79,15 +79,15 @@ func TestLoadConfigWithValidation(t *testing.T) { configContent := ` fileSizeLimit: 100 ignoreDirectories: - - node_modules - - "" - - .git + - node_modules + - "" + - .git ` tempDir := t.TempDir() configFile := tempDir + "/config.yaml" - err := os.WriteFile(configFile, []byte(configContent), 0o644) + err := os.WriteFile(configFile, []byte(configContent), 0o600) if err != nil { t.Fatalf("Failed to write config file: %v", err) } @@ -104,7 +104,10 @@ ignoreDirectories: t.Errorf("Expected default file size limit after validation failure, got %d", config.GetFileSizeLimit()) } if containsString(config.GetIgnoredDirectories(), "") { - t.Errorf("Expected ignored directories not to contain empty string after validation failure, got %v", config.GetIgnoredDirectories()) + t.Errorf( + "Expected ignored directories not to contain empty string after validation failure, got %v", + config.GetIgnoredDirectories(), + ) } } @@ -117,4 +120,4 @@ func containsString(slice []string, item string) bool { } } return false -} \ No newline at end of file +} diff --git a/config/validation.go b/config/validation.go index ed13319..373598d 100644 --- a/config/validation.go +++ b/config/validation.go @@ -6,240 +6,532 @@ import ( "github.com/spf13/viper" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) -// ValidateConfig validates the loaded configuration. -func ValidateConfig() error { - var validationErrors []string - - // Validate file size limit +// validateFileSizeLimit validates the file size limit configuration. +func validateFileSizeLimit() []string { + var errors []string fileSizeLimit := viper.GetInt64("fileSizeLimit") if fileSizeLimit < MinFileSizeLimit { - validationErrors = append(validationErrors, fmt.Sprintf("fileSizeLimit (%d) is below minimum (%d)", fileSizeLimit, MinFileSizeLimit)) + errors = append( + errors, + fmt.Sprintf("fileSizeLimit (%d) is below minimum (%d)", fileSizeLimit, MinFileSizeLimit), + ) } if fileSizeLimit > MaxFileSizeLimit { - validationErrors = append(validationErrors, fmt.Sprintf("fileSizeLimit (%d) exceeds maximum (%d)", fileSizeLimit, MaxFileSizeLimit)) + errors = append( + errors, + fmt.Sprintf("fileSizeLimit (%d) exceeds maximum (%d)", fileSizeLimit, MaxFileSizeLimit), + ) } + return errors +} - // Validate ignore directories +// validateIgnoreDirectories validates the ignore directories configuration. +func validateIgnoreDirectories() []string { + var errors []string ignoreDirectories := viper.GetStringSlice("ignoreDirectories") for i, dir := range ignoreDirectories { dir = strings.TrimSpace(dir) if dir == "" { - validationErrors = append(validationErrors, fmt.Sprintf("ignoreDirectories[%d] is empty", i)) + errors = append(errors, fmt.Sprintf("ignoreDirectories[%d] is empty", i)) continue } if strings.Contains(dir, "/") { - validationErrors = append(validationErrors, fmt.Sprintf("ignoreDirectories[%d] (%s) contains path separator - only directory names are allowed", i, dir)) + errors = append( + errors, + fmt.Sprintf( + "ignoreDirectories[%d] (%s) contains path separator - only directory names are allowed", + i, + dir, + ), + ) } if strings.HasPrefix(dir, ".") && dir != ".git" && dir != ".vscode" && dir != ".idea" { - validationErrors = append(validationErrors, fmt.Sprintf("ignoreDirectories[%d] (%s) starts with dot - this may cause unexpected behavior", i, dir)) + errors = append( + errors, + fmt.Sprintf("ignoreDirectories[%d] (%s) starts with dot - this may cause unexpected behavior", i, dir), + ) } } + return errors +} - // Validate supported output formats if configured +// validateSupportedFormats validates the supported output formats configuration. +func validateSupportedFormats() []string { + var errors []string if viper.IsSet("supportedFormats") { supportedFormats := viper.GetStringSlice("supportedFormats") validFormats := map[string]bool{"json": true, "yaml": true, "markdown": true} for i, format := range supportedFormats { format = strings.ToLower(strings.TrimSpace(format)) if !validFormats[format] { - validationErrors = append(validationErrors, fmt.Sprintf("supportedFormats[%d] (%s) is not a valid format (json, yaml, markdown)", i, format)) + errors = append( + errors, + fmt.Sprintf("supportedFormats[%d] (%s) is not a valid format (json, yaml, markdown)", i, format), + ) } } } + return errors +} - // Validate concurrency settings if configured +// validateConcurrencySettings validates the concurrency settings configuration. +func validateConcurrencySettings() []string { + var errors []string if viper.IsSet("maxConcurrency") { maxConcurrency := viper.GetInt("maxConcurrency") if maxConcurrency < 1 { - validationErrors = append(validationErrors, fmt.Sprintf("maxConcurrency (%d) must be at least 1", maxConcurrency)) + errors = append( + errors, + fmt.Sprintf("maxConcurrency (%d) must be at least 1", maxConcurrency), + ) } if maxConcurrency > 100 { - validationErrors = append(validationErrors, fmt.Sprintf("maxConcurrency (%d) is unreasonably high (max 100)", maxConcurrency)) + errors = append( + errors, + fmt.Sprintf("maxConcurrency (%d) is unreasonably high (max 100)", maxConcurrency), + ) } } + return errors +} - // Validate file patterns if configured +// validateFilePatterns validates the file patterns configuration. +func validateFilePatterns() []string { + var errors []string if viper.IsSet("filePatterns") { filePatterns := viper.GetStringSlice("filePatterns") for i, pattern := range filePatterns { pattern = strings.TrimSpace(pattern) if pattern == "" { - validationErrors = append(validationErrors, fmt.Sprintf("filePatterns[%d] is empty", i)) + errors = append(errors, fmt.Sprintf("filePatterns[%d] is empty", i)) continue } // Basic validation - patterns should contain at least one alphanumeric character if !strings.ContainsAny(pattern, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { - validationErrors = append(validationErrors, fmt.Sprintf("filePatterns[%d] (%s) appears to be invalid", i, pattern)) + errors = append( + errors, + fmt.Sprintf("filePatterns[%d] (%s) appears to be invalid", i, pattern), + ) } } } + return errors +} - // Validate FileTypeRegistry configuration - if viper.IsSet("fileTypes.customImageExtensions") { - customImages := viper.GetStringSlice("fileTypes.customImageExtensions") - for i, ext := range customImages { - ext = strings.TrimSpace(ext) - if ext == "" { - validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customImageExtensions[%d] is empty", i)) - continue - } - if !strings.HasPrefix(ext, ".") { - validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customImageExtensions[%d] (%s) must start with a dot", i, ext)) - } - } +// validateFileTypes validates the FileTypeRegistry configuration. +// validateCustomImageExtensions validates custom image extensions configuration. +func validateCustomImageExtensions() []string { + var errors []string + if !viper.IsSet("fileTypes.customImageExtensions") { + return errors } - if viper.IsSet("fileTypes.customBinaryExtensions") { - customBinary := viper.GetStringSlice("fileTypes.customBinaryExtensions") - for i, ext := range customBinary { - ext = strings.TrimSpace(ext) - if ext == "" { - validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customBinaryExtensions[%d] is empty", i)) - continue - } - if !strings.HasPrefix(ext, ".") { - validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customBinaryExtensions[%d] (%s) must start with a dot", i, ext)) - } + customImages := viper.GetStringSlice("fileTypes.customImageExtensions") + for i, ext := range customImages { + ext = strings.TrimSpace(ext) + if ext == "" { + errors = append( + errors, + fmt.Sprintf("fileTypes.customImageExtensions[%d] is empty", i), + ) + continue } + if !strings.HasPrefix(ext, ".") { + errors = append( + errors, + fmt.Sprintf("fileTypes.customImageExtensions[%d] (%s) must start with a dot", i, ext), + ) + } + } + return errors +} + +// validateCustomBinaryExtensions validates custom binary extensions configuration. +func validateCustomBinaryExtensions() []string { + var errors []string + if !viper.IsSet("fileTypes.customBinaryExtensions") { + return errors } - if viper.IsSet("fileTypes.customLanguages") { - customLangs := viper.GetStringMapString("fileTypes.customLanguages") - for ext, lang := range customLangs { - ext = strings.TrimSpace(ext) - lang = strings.TrimSpace(lang) - if ext == "" { - validationErrors = append(validationErrors, "fileTypes.customLanguages contains empty extension key") - continue - } - if !strings.HasPrefix(ext, ".") { - validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customLanguages extension (%s) must start with a dot", ext)) - } - if lang == "" { - validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customLanguages[%s] has empty language value", ext)) - } + customBinary := viper.GetStringSlice("fileTypes.customBinaryExtensions") + for i, ext := range customBinary { + ext = strings.TrimSpace(ext) + if ext == "" { + errors = append( + errors, + fmt.Sprintf("fileTypes.customBinaryExtensions[%d] is empty", i), + ) + continue } + if !strings.HasPrefix(ext, ".") { + errors = append( + errors, + fmt.Sprintf("fileTypes.customBinaryExtensions[%d] (%s) must start with a dot", i, ext), + ) + } + } + return errors +} + +// validateCustomLanguages validates custom languages configuration. +func validateCustomLanguages() []string { + var errors []string + if !viper.IsSet("fileTypes.customLanguages") { + return errors } - // Validate back-pressure configuration - if viper.IsSet("backpressure.maxPendingFiles") { - maxPendingFiles := viper.GetInt("backpressure.maxPendingFiles") - if maxPendingFiles < 1 { - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingFiles (%d) must be at least 1", maxPendingFiles)) + customLangs := viper.GetStringMapString("fileTypes.customLanguages") + for ext, lang := range customLangs { + ext = strings.TrimSpace(ext) + lang = strings.TrimSpace(lang) + if ext == "" { + errors = append(errors, "fileTypes.customLanguages contains empty extension key") + continue } - if maxPendingFiles > 100000 { - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingFiles (%d) is unreasonably high (max 100000)", maxPendingFiles)) + if !strings.HasPrefix(ext, ".") { + errors = append( + errors, + fmt.Sprintf("fileTypes.customLanguages extension (%s) must start with a dot", ext), + ) } + if lang == "" { + errors = append( + errors, + fmt.Sprintf("fileTypes.customLanguages[%s] has empty language value", ext), + ) + } + } + return errors +} + +// validateFileTypes validates the FileTypeRegistry configuration. +func validateFileTypes() []string { + var errors []string + errors = append(errors, validateCustomImageExtensions()...) + errors = append(errors, validateCustomBinaryExtensions()...) + errors = append(errors, validateCustomLanguages()...) + return errors +} + +// validateBackpressureConfig validates the back-pressure configuration. +// validateBackpressureMaxPendingFiles validates max pending files configuration. +func validateBackpressureMaxPendingFiles() []string { + var errors []string + if !viper.IsSet("backpressure.maxPendingFiles") { + return errors } - if viper.IsSet("backpressure.maxPendingWrites") { - maxPendingWrites := viper.GetInt("backpressure.maxPendingWrites") - if maxPendingWrites < 1 { - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingWrites (%d) must be at least 1", maxPendingWrites)) - } - if maxPendingWrites > 10000 { - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingWrites (%d) is unreasonably high (max 10000)", maxPendingWrites)) - } + maxPendingFiles := viper.GetInt("backpressure.maxPendingFiles") + if maxPendingFiles < 1 { + errors = append( + errors, + fmt.Sprintf("backpressure.maxPendingFiles (%d) must be at least 1", maxPendingFiles), + ) + } + if maxPendingFiles > 100000 { + errors = append( + errors, + fmt.Sprintf("backpressure.maxPendingFiles (%d) is unreasonably high (max 100000)", maxPendingFiles), + ) + } + return errors +} + +// validateBackpressureMaxPendingWrites validates max pending writes configuration. +func validateBackpressureMaxPendingWrites() []string { + var errors []string + if !viper.IsSet("backpressure.maxPendingWrites") { + return errors } - if viper.IsSet("backpressure.maxMemoryUsage") { - maxMemoryUsage := viper.GetInt64("backpressure.maxMemoryUsage") - if maxMemoryUsage < 1048576 { // 1MB minimum - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxMemoryUsage (%d) must be at least 1MB (1048576 bytes)", maxMemoryUsage)) - } - if maxMemoryUsage > 10737418240 { // 10GB maximum - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxMemoryUsage (%d) is unreasonably high (max 10GB)", maxMemoryUsage)) - } + maxPendingWrites := viper.GetInt("backpressure.maxPendingWrites") + if maxPendingWrites < 1 { + errors = append( + errors, + fmt.Sprintf("backpressure.maxPendingWrites (%d) must be at least 1", maxPendingWrites), + ) + } + if maxPendingWrites > 10000 { + errors = append( + errors, + fmt.Sprintf("backpressure.maxPendingWrites (%d) is unreasonably high (max 10000)", maxPendingWrites), + ) + } + return errors +} + +// validateBackpressureMaxMemoryUsage validates max memory usage configuration. +func validateBackpressureMaxMemoryUsage() []string { + var errors []string + if !viper.IsSet("backpressure.maxMemoryUsage") { + return errors } - if viper.IsSet("backpressure.memoryCheckInterval") { - interval := viper.GetInt("backpressure.memoryCheckInterval") - if interval < 1 { - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.memoryCheckInterval (%d) must be at least 1", interval)) - } - if interval > 100000 { - validationErrors = append(validationErrors, fmt.Sprintf("backpressure.memoryCheckInterval (%d) is unreasonably high (max 100000)", interval)) - } + maxMemoryUsage := viper.GetInt64("backpressure.maxMemoryUsage") + if maxMemoryUsage < 1048576 { // 1MB minimum + errors = append( + errors, + fmt.Sprintf("backpressure.maxMemoryUsage (%d) must be at least 1MB (1048576 bytes)", maxMemoryUsage), + ) + } + if maxMemoryUsage > 104857600 { // 100MB maximum + errors = append( + errors, + fmt.Sprintf("backpressure.maxMemoryUsage (%d) is unreasonably high (max 100MB)", maxMemoryUsage), + ) + } + return errors +} + +// validateBackpressureMemoryCheckInterval validates memory check interval configuration. +func validateBackpressureMemoryCheckInterval() []string { + var errors []string + if !viper.IsSet("backpressure.memoryCheckInterval") { + return errors } - // Validate resource limits configuration - if viper.IsSet("resourceLimits.maxFiles") { - maxFiles := viper.GetInt("resourceLimits.maxFiles") - if maxFiles < MinMaxFiles { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxFiles (%d) must be at least %d", maxFiles, MinMaxFiles)) - } - if maxFiles > MaxMaxFiles { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxFiles (%d) exceeds maximum (%d)", maxFiles, MaxMaxFiles)) - } + interval := viper.GetInt("backpressure.memoryCheckInterval") + if interval < 1 { + errors = append( + errors, + fmt.Sprintf("backpressure.memoryCheckInterval (%d) must be at least 1", interval), + ) + } + if interval > 100000 { + errors = append( + errors, + fmt.Sprintf("backpressure.memoryCheckInterval (%d) is unreasonably high (max 100000)", interval), + ) + } + return errors +} + +// validateBackpressureConfig validates the back-pressure configuration. +func validateBackpressureConfig() []string { + var errors []string + errors = append(errors, validateBackpressureMaxPendingFiles()...) + errors = append(errors, validateBackpressureMaxPendingWrites()...) + errors = append(errors, validateBackpressureMaxMemoryUsage()...) + errors = append(errors, validateBackpressureMemoryCheckInterval()...) + return errors +} + +// validateResourceLimits validates the resource limits configuration. +// validateResourceLimitsMaxFiles validates max files configuration. +func validateResourceLimitsMaxFiles() []string { + var errors []string + if !viper.IsSet("resourceLimits.maxFiles") { + return errors } - if viper.IsSet("resourceLimits.maxTotalSize") { - maxTotalSize := viper.GetInt64("resourceLimits.maxTotalSize") - if maxTotalSize < MinMaxTotalSize { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxTotalSize (%d) must be at least %d", maxTotalSize, MinMaxTotalSize)) - } - if maxTotalSize > MaxMaxTotalSize { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxTotalSize (%d) exceeds maximum (%d)", maxTotalSize, MaxMaxTotalSize)) - } + maxFiles := viper.GetInt("resourceLimits.maxFiles") + if maxFiles < MinMaxFiles { + errors = append( + errors, + fmt.Sprintf("resourceLimits.maxFiles (%d) must be at least %d", maxFiles, MinMaxFiles), + ) } + if maxFiles > MaxMaxFiles { + errors = append( + errors, + fmt.Sprintf("resourceLimits.maxFiles (%d) exceeds maximum (%d)", maxFiles, MaxMaxFiles), + ) + } + return errors +} + +// validateResourceLimitsMaxTotalSize validates max total size configuration. +func validateResourceLimitsMaxTotalSize() []string { + var errors []string + if !viper.IsSet("resourceLimits.maxTotalSize") { + return errors + } + + maxTotalSize := viper.GetInt64("resourceLimits.maxTotalSize") + if maxTotalSize < MinMaxTotalSize { + errors = append( + errors, + fmt.Sprintf("resourceLimits.maxTotalSize (%d) must be at least %d", maxTotalSize, MinMaxTotalSize), + ) + } + if maxTotalSize > MaxMaxTotalSize { + errors = append( + errors, + fmt.Sprintf("resourceLimits.maxTotalSize (%d) exceeds maximum (%d)", maxTotalSize, MaxMaxTotalSize), + ) + } + return errors +} + +// validateResourceLimitsTimeouts validates timeout configurations. +func validateResourceLimitsTimeouts() []string { + var errors []string if viper.IsSet("resourceLimits.fileProcessingTimeoutSec") { timeout := viper.GetInt("resourceLimits.fileProcessingTimeoutSec") if timeout < MinFileProcessingTimeoutSec { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.fileProcessingTimeoutSec (%d) must be at least %d", timeout, MinFileProcessingTimeoutSec)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.fileProcessingTimeoutSec (%d) must be at least %d", + timeout, + MinFileProcessingTimeoutSec, + ), + ) } if timeout > MaxFileProcessingTimeoutSec { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.fileProcessingTimeoutSec (%d) exceeds maximum (%d)", timeout, MaxFileProcessingTimeoutSec)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.fileProcessingTimeoutSec (%d) exceeds maximum (%d)", + timeout, + MaxFileProcessingTimeoutSec, + ), + ) } } if viper.IsSet("resourceLimits.overallTimeoutSec") { timeout := viper.GetInt("resourceLimits.overallTimeoutSec") if timeout < MinOverallTimeoutSec { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) must be at least %d", timeout, MinOverallTimeoutSec)) + errors = append( + errors, + fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) must be at least %d", timeout, MinOverallTimeoutSec), + ) } if timeout > MaxOverallTimeoutSec { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) exceeds maximum (%d)", timeout, MaxOverallTimeoutSec)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.overallTimeoutSec (%d) exceeds maximum (%d)", + timeout, + MaxOverallTimeoutSec, + ), + ) } } + return errors +} + +// validateResourceLimitsConcurrency validates concurrency configurations. +func validateResourceLimitsConcurrency() []string { + var errors []string + if viper.IsSet("resourceLimits.maxConcurrentReads") { maxReads := viper.GetInt("resourceLimits.maxConcurrentReads") if maxReads < MinMaxConcurrentReads { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) must be at least %d", maxReads, MinMaxConcurrentReads)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.maxConcurrentReads (%d) must be at least %d", + maxReads, + MinMaxConcurrentReads, + ), + ) } if maxReads > MaxMaxConcurrentReads { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) exceeds maximum (%d)", maxReads, MaxMaxConcurrentReads)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.maxConcurrentReads (%d) exceeds maximum (%d)", + maxReads, + MaxMaxConcurrentReads, + ), + ) } } if viper.IsSet("resourceLimits.rateLimitFilesPerSec") { rateLimit := viper.GetInt("resourceLimits.rateLimitFilesPerSec") if rateLimit < MinRateLimitFilesPerSec { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) must be at least %d", rateLimit, MinRateLimitFilesPerSec)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.rateLimitFilesPerSec (%d) must be at least %d", + rateLimit, + MinRateLimitFilesPerSec, + ), + ) } if rateLimit > MaxRateLimitFilesPerSec { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) exceeds maximum (%d)", rateLimit, MaxRateLimitFilesPerSec)) + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.rateLimitFilesPerSec (%d) exceeds maximum (%d)", + rateLimit, + MaxRateLimitFilesPerSec, + ), + ) } } - if viper.IsSet("resourceLimits.hardMemoryLimitMB") { - memLimit := viper.GetInt("resourceLimits.hardMemoryLimitMB") - if memLimit < MinHardMemoryLimitMB { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) must be at least %d", memLimit, MinHardMemoryLimitMB)) - } - if memLimit > MaxHardMemoryLimitMB { - validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) exceeds maximum (%d)", memLimit, MaxHardMemoryLimitMB)) - } + return errors +} + +// validateResourceLimitsMemory validates memory limit configuration. +func validateResourceLimitsMemory() []string { + var errors []string + if !viper.IsSet("resourceLimits.hardMemoryLimitMB") { + return errors } + memLimit := viper.GetInt("resourceLimits.hardMemoryLimitMB") + if memLimit < MinHardMemoryLimitMB { + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.hardMemoryLimitMB (%d) must be at least %d", + memLimit, + MinHardMemoryLimitMB, + ), + ) + } + if memLimit > MaxHardMemoryLimitMB { + errors = append( + errors, + fmt.Sprintf( + "resourceLimits.hardMemoryLimitMB (%d) exceeds maximum (%d)", + memLimit, + MaxHardMemoryLimitMB, + ), + ) + } + return errors +} + +// validateResourceLimits validates the resource limits configuration. +func validateResourceLimits() []string { + var errors []string + errors = append(errors, validateResourceLimitsMaxFiles()...) + errors = append(errors, validateResourceLimitsMaxTotalSize()...) + errors = append(errors, validateResourceLimitsTimeouts()...) + errors = append(errors, validateResourceLimitsConcurrency()...) + errors = append(errors, validateResourceLimitsMemory()...) + return errors +} + +// ValidateConfig validates the loaded configuration. +func ValidateConfig() error { + var validationErrors []string + + // Collect validation errors from all validation helpers + validationErrors = append(validationErrors, validateFileSizeLimit()...) + validationErrors = append(validationErrors, validateIgnoreDirectories()...) + validationErrors = append(validationErrors, validateSupportedFormats()...) + validationErrors = append(validationErrors, validateConcurrencySettings()...) + validationErrors = append(validationErrors, validateFilePatterns()...) + validationErrors = append(validationErrors, validateFileTypes()...) + validationErrors = append(validationErrors, validateBackpressureConfig()...) + validationErrors = append(validationErrors, validateResourceLimits()...) + if len(validationErrors) > 0 { - return utils.NewStructuredError( - utils.ErrorTypeConfiguration, - utils.CodeConfigValidation, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeConfiguration, + gibidiutils.CodeConfigValidation, "configuration validation failed: "+strings.Join(validationErrors, "; "), "", map[string]interface{}{"validation_errors": validationErrors}, @@ -253,9 +545,9 @@ func ValidateConfig() error { func ValidateFileSize(size int64) error { limit := GetFileSizeLimit() if size > limit { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeValidationSize, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationSize, fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", size, limit), "", map[string]interface{}{"file_size": size, "size_limit": limit}, @@ -267,9 +559,9 @@ func ValidateFileSize(size int64) error { // ValidateOutputFormat checks if an output format is valid. func ValidateOutputFormat(format string) error { if !IsValidFormat(format) { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeValidationFormat, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, fmt.Sprintf("unsupported output format: %s (supported: json, yaml, markdown)", format), "", map[string]interface{}{"format": format}, @@ -281,9 +573,9 @@ func ValidateOutputFormat(format string) error { // ValidateConcurrency checks if a concurrency level is valid. func ValidateConcurrency(concurrency int) error { if concurrency < 1 { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeValidationFormat, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, fmt.Sprintf("concurrency (%d) must be at least 1", concurrency), "", map[string]interface{}{"concurrency": concurrency}, @@ -293,9 +585,9 @@ func ValidateConcurrency(concurrency int) error { if viper.IsSet("maxConcurrency") { maxConcurrency := GetMaxConcurrency() if concurrency > maxConcurrency { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeValidationFormat, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, fmt.Sprintf("concurrency (%d) exceeds maximum (%d)", concurrency, maxConcurrency), "", map[string]interface{}{"concurrency": concurrency, "max_concurrency": maxConcurrency}, @@ -304,4 +596,4 @@ func ValidateConcurrency(concurrency int) error { } return nil -} \ No newline at end of file +} diff --git a/config/validation_test.go b/config/validation_test.go index 9b3cfb4..13bedc8 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -1,13 +1,14 @@ package config_test import ( + "errors" "strings" "testing" "github.com/spf13/viper" "github.com/ivuorinen/gibidify/config" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // TestValidateConfig tests the configuration validation functionality. @@ -112,21 +113,19 @@ func TestValidateConfig(t *testing.T) { } // Check that it's a structured error - var structErr *utils.StructuredError + var structErr *gibidiutils.StructuredError if !errorAs(err, &structErr) { t.Errorf("Expected structured error, got %T", err) return } - if structErr.Type != utils.ErrorTypeConfiguration { - t.Errorf("Expected error type %v, got %v", utils.ErrorTypeConfiguration, structErr.Type) + if structErr.Type != gibidiutils.ErrorTypeConfiguration { + t.Errorf("Expected error type %v, got %v", gibidiutils.ErrorTypeConfiguration, structErr.Type) } - if structErr.Code != utils.CodeConfigValidation { - t.Errorf("Expected error code %v, got %v", utils.CodeConfigValidation, structErr.Code) - } - } else { - if err != nil { - t.Errorf("Expected no error but got: %v", err) + if structErr.Code != gibidiutils.CodeConfigValidation { + t.Errorf("Expected error code %v, got %v", gibidiutils.CodeConfigValidation, structErr.Code) } + } else if err != nil { + t.Errorf("Expected no error but got: %v", err) } }) } @@ -235,11 +234,12 @@ func errorAs(err error, target interface{}) bool { if err == nil { return false } - if structErr, ok := err.(*utils.StructuredError); ok { - if ptr, ok := target.(**utils.StructuredError); ok { + var structErr *gibidiutils.StructuredError + if errors.As(err, &structErr) { + if ptr, ok := target.(**gibidiutils.StructuredError); ok { *ptr = structErr return true } } return false -} \ No newline at end of file +} diff --git a/fileproc/backpressure.go b/fileproc/backpressure.go index 733a271..379bacc 100644 --- a/fileproc/backpressure.go +++ b/fileproc/backpressure.go @@ -3,6 +3,7 @@ package fileproc import ( "context" + "math" "runtime" "sync" "sync/atomic" @@ -11,6 +12,7 @@ import ( "github.com/sirupsen/logrus" "github.com/ivuorinen/gibidify/config" + "github.com/ivuorinen/gibidify/gibidiutils" ) // BackpressureManager manages memory usage and applies back-pressure when needed. @@ -59,21 +61,22 @@ func (bp *BackpressureManager) CreateChannels() (chan string, chan WriteRequest) } // ShouldApplyBackpressure checks if back-pressure should be applied. -func (bp *BackpressureManager) ShouldApplyBackpressure(ctx context.Context) bool { +func (bp *BackpressureManager) ShouldApplyBackpressure(_ context.Context) bool { if !bp.enabled { return false } // Check if we should evaluate memory usage filesProcessed := atomic.AddInt64(&bp.filesProcessed, 1) - if int(filesProcessed)%bp.memoryCheckInterval != 0 { + // Avoid divide by zero - if interval is 0, check every file + if bp.memoryCheckInterval > 0 && int(filesProcessed)%bp.memoryCheckInterval != 0 { return false } // Get current memory usage var m runtime.MemStats runtime.ReadMemStats(&m) - currentMemory := int64(m.Alloc) + currentMemory := gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, math.MaxInt64) bp.mu.Lock() defer bp.mu.Unlock() @@ -133,7 +136,7 @@ func (bp *BackpressureManager) GetStats() BackpressureStats { return BackpressureStats{ Enabled: bp.enabled, FilesProcessed: atomic.LoadInt64(&bp.filesProcessed), - CurrentMemoryUsage: int64(m.Alloc), + CurrentMemoryUsage: gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, math.MaxInt64), MaxMemoryUsage: bp.maxMemoryUsage, MemoryWarningActive: bp.memoryWarningLogged, LastMemoryCheck: bp.lastMemoryCheck, @@ -160,8 +163,8 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c return } - // Check if file channel is getting full (>90% capacity) - if len(fileCh) > bp.maxPendingFiles*9/10 { + // Check if file channel is getting full (>=90% capacity) + if bp.maxPendingFiles > 0 && len(fileCh) >= bp.maxPendingFiles*9/10 { logrus.Debugf("File channel is %d%% full, waiting for space", len(fileCh)*100/bp.maxPendingFiles) // Wait a bit for the channel to drain @@ -172,8 +175,8 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c } } - // Check if write channel is getting full (>90% capacity) - if len(writeCh) > bp.maxPendingWrites*9/10 { + // Check if write channel is getting full (>=90% capacity) + if bp.maxPendingWrites > 0 && len(writeCh) >= bp.maxPendingWrites*9/10 { logrus.Debugf("Write channel is %d%% full, waiting for space", len(writeCh)*100/bp.maxPendingWrites) // Wait a bit for the channel to drain diff --git a/fileproc/backpressure_behavior_test.go b/fileproc/backpressure_behavior_test.go new file mode 100644 index 0000000..3308b8e --- /dev/null +++ b/fileproc/backpressure_behavior_test.go @@ -0,0 +1,177 @@ +package fileproc + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBackpressureManagerShouldApplyBackpressure(t *testing.T) { + ctx := context.Background() + + t.Run("returns false when disabled", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = false + + shouldApply := bm.ShouldApplyBackpressure(ctx) + assert.False(t, shouldApply) + }) + + t.Run("checks memory at intervals", func(_ *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true + bm.memoryCheckInterval = 10 + + // Should not check memory on most calls + for i := 1; i < 10; i++ { + shouldApply := bm.ShouldApplyBackpressure(ctx) + // Can't predict result, but shouldn't panic + _ = shouldApply + } + + // Should check memory on 10th call + shouldApply := bm.ShouldApplyBackpressure(ctx) + // Result depends on actual memory usage + _ = shouldApply + }) + + t.Run("detects high memory usage", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true + bm.memoryCheckInterval = 1 + bm.maxMemoryUsage = 1 // Set very low limit to trigger + + shouldApply := bm.ShouldApplyBackpressure(ctx) + // Should detect high memory usage + assert.True(t, shouldApply) + }) +} + +func TestBackpressureManagerApplyBackpressure(t *testing.T) { + ctx := context.Background() + + t.Run("does nothing when disabled", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = false + + // Use a channel to verify the function returns quickly + done := make(chan struct{}) + go func() { + bm.ApplyBackpressure(ctx) + close(done) + }() + + // Should complete quickly when disabled + select { + case <-done: + // Success - function returned + case <-time.After(50 * time.Millisecond): + t.Fatal("ApplyBackpressure did not return quickly when disabled") + } + }) + + t.Run("applies delay when enabled", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true + + // Use a channel to verify the function blocks for some time + done := make(chan struct{}) + started := make(chan struct{}) + go func() { + close(started) + bm.ApplyBackpressure(ctx) + close(done) + }() + + // Wait for goroutine to start + <-started + + // Should NOT complete immediately - verify it blocks for at least 5ms + select { + case <-done: + t.Fatal("ApplyBackpressure returned too quickly when enabled") + case <-time.After(5 * time.Millisecond): + // Good - it's blocking as expected + } + + // Now wait for it to complete (should finish within reasonable time) + select { + case <-done: + // Success - function eventually returned + case <-time.After(500 * time.Millisecond): + t.Fatal("ApplyBackpressure did not complete within timeout") + } + }) + + t.Run("respects context cancellation", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + start := time.Now() + bm.ApplyBackpressure(ctx) + duration := time.Since(start) + + // Should return quickly when context is cancelled + assert.Less(t, duration, 5*time.Millisecond) + }) +} + +func TestBackpressureManagerLogBackpressureInfo(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true // Ensure enabled so filesProcessed is incremented + + // Apply some operations + ctx := context.Background() + bm.ShouldApplyBackpressure(ctx) + bm.ApplyBackpressure(ctx) + + // This should not panic + bm.LogBackpressureInfo() + + stats := bm.GetStats() + assert.Greater(t, stats.FilesProcessed, int64(0)) +} + +func TestBackpressureManagerMemoryLimiting(t *testing.T) { + t.Run("triggers on low memory limit", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true + bm.memoryCheckInterval = 1 // Check every file + bm.maxMemoryUsage = 1 // Very low limit to guarantee trigger + + ctx := context.Background() + + // Should detect memory over limit + shouldApply := bm.ShouldApplyBackpressure(ctx) + assert.True(t, shouldApply) + stats := bm.GetStats() + assert.True(t, stats.MemoryWarningActive) + }) + + t.Run("resets warning when memory normalizes", func(t *testing.T) { + bm := NewBackpressureManager() + bm.enabled = true + bm.memoryCheckInterval = 1 + // Simulate warning by first triggering high memory usage + bm.maxMemoryUsage = 1 // Very low to trigger warning + ctx := context.Background() + _ = bm.ShouldApplyBackpressure(ctx) + stats := bm.GetStats() + assert.True(t, stats.MemoryWarningActive) + + // Now set high limit so we're under it + bm.maxMemoryUsage = 1024 * 1024 * 1024 * 10 // 10GB + + shouldApply := bm.ShouldApplyBackpressure(ctx) + assert.False(t, shouldApply) + + // Warning should be reset (via public API) + stats = bm.GetStats() + assert.False(t, stats.MemoryWarningActive) + }) +} diff --git a/fileproc/backpressure_channels_test.go b/fileproc/backpressure_channels_test.go new file mode 100644 index 0000000..2882918 --- /dev/null +++ b/fileproc/backpressure_channels_test.go @@ -0,0 +1,262 @@ +package fileproc + +import ( + "context" + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +const ( + // CI-safe timeout constants + fastOpTimeout = 100 * time.Millisecond // Operations that should complete quickly + slowOpMinTime = 10 * time.Millisecond // Minimum time for blocking operations +) + +// cleanupViperConfig is a test helper that captures and restores viper configuration. +// It takes a testing.T and a list of config keys to save/restore. +// Returns a cleanup function that should be called via t.Cleanup. +func cleanupViperConfig(t *testing.T, keys ...string) { + t.Helper() + // Capture original values + origValues := make(map[string]interface{}) + for _, key := range keys { + origValues[key] = viper.Get(key) + } + // Register cleanup to restore values + t.Cleanup(func() { + for key, val := range origValues { + if val != nil { + viper.Set(key, val) + } + } + }) +} + +func TestBackpressureManagerCreateChannels(t *testing.T) { + t.Run("creates buffered channels when enabled", func(t *testing.T) { + // Capture and restore viper config + cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles, testBackpressureMaxWrites) + + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxFiles, 10) + viper.Set(testBackpressureMaxWrites, 10) + bm := NewBackpressureManager() + + fileCh, writeCh := bm.CreateChannels() + assert.NotNil(t, fileCh) + assert.NotNil(t, writeCh) + + // Test that channels have buffer capacity + assert.Greater(t, cap(fileCh), 0) + assert.Greater(t, cap(writeCh), 0) + + // Test sending and receiving + fileCh <- "test.go" + val := <-fileCh + assert.Equal(t, "test.go", val) + + writeCh <- WriteRequest{Content: "test content"} + writeReq := <-writeCh + assert.Equal(t, "test content", writeReq.Content) + + close(fileCh) + close(writeCh) + }) + + t.Run("creates unbuffered channels when disabled", func(t *testing.T) { + // Use viper to configure instead of direct field access + cleanupViperConfig(t, testBackpressureEnabled) + + viper.Set(testBackpressureEnabled, false) + bm := NewBackpressureManager() + + fileCh, writeCh := bm.CreateChannels() + assert.NotNil(t, fileCh) + assert.NotNil(t, writeCh) + + // Unbuffered channels have capacity 0 + assert.Equal(t, 0, cap(fileCh)) + assert.Equal(t, 0, cap(writeCh)) + + close(fileCh) + close(writeCh) + }) +} + +func TestBackpressureManagerWaitForChannelSpace(t *testing.T) { + t.Run("does nothing when disabled", func(t *testing.T) { + // Use viper to configure instead of direct field access + cleanupViperConfig(t, testBackpressureEnabled) + + viper.Set(testBackpressureEnabled, false) + bm := NewBackpressureManager() + + fileCh := make(chan string, 1) + writeCh := make(chan WriteRequest, 1) + + // Use context with timeout instead of measuring elapsed time + ctx, cancel := context.WithTimeout(context.Background(), fastOpTimeout) + defer cancel() + + done := make(chan struct{}) + go func() { + bm.WaitForChannelSpace(ctx, fileCh, writeCh) + close(done) + }() + + // Should return immediately (before timeout) + select { + case <-done: + // Success - operation completed quickly + case <-ctx.Done(): + t.Fatal("WaitForChannelSpace should return immediately when disabled") + } + + close(fileCh) + close(writeCh) + }) + + t.Run("waits when file channel is nearly full", func(t *testing.T) { + // Use viper to configure instead of direct field access + cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles) + + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxFiles, 10) + bm := NewBackpressureManager() + + // Create channel with exact capacity + fileCh := make(chan string, 10) + writeCh := make(chan WriteRequest, 10) + + // Fill file channel to >90% (with minimum of 1) + target := max(1, int(float64(cap(fileCh))*0.9)) + for i := 0; i < target; i++ { + fileCh <- "file.txt" + } + + // Test that it blocks by verifying it doesn't complete immediately + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan struct{}) + start := time.Now() + go func() { + bm.WaitForChannelSpace(ctx, fileCh, writeCh) + close(done) + }() + + // Verify it doesn't complete immediately (within first millisecond) + select { + case <-done: + t.Fatal("WaitForChannelSpace should block when channel is nearly full") + case <-time.After(1 * time.Millisecond): + // Good - it's blocking as expected + } + + // Wait for it to complete + <-done + duration := time.Since(start) + // Just verify it took some measurable time (very lenient for CI) + assert.GreaterOrEqual(t, duration, 1*time.Millisecond) + + // Clean up + for i := 0; i < target; i++ { + <-fileCh + } + close(fileCh) + close(writeCh) + }) + + t.Run("waits when write channel is nearly full", func(t *testing.T) { + // Use viper to configure instead of direct field access + cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxWrites) + + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxWrites, 10) + bm := NewBackpressureManager() + + fileCh := make(chan string, 10) + writeCh := make(chan WriteRequest, 10) + + // Fill write channel to >90% (with minimum of 1) + target := max(1, int(float64(cap(writeCh))*0.9)) + for i := 0; i < target; i++ { + writeCh <- WriteRequest{} + } + + // Test that it blocks by verifying it doesn't complete immediately + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan struct{}) + start := time.Now() + go func() { + bm.WaitForChannelSpace(ctx, fileCh, writeCh) + close(done) + }() + + // Verify it doesn't complete immediately (within first millisecond) + select { + case <-done: + t.Fatal("WaitForChannelSpace should block when channel is nearly full") + case <-time.After(1 * time.Millisecond): + // Good - it's blocking as expected + } + + // Wait for it to complete + <-done + duration := time.Since(start) + // Just verify it took some measurable time (very lenient for CI) + assert.GreaterOrEqual(t, duration, 1*time.Millisecond) + + // Clean up + for i := 0; i < target; i++ { + <-writeCh + } + close(fileCh) + close(writeCh) + }) + + t.Run("respects context cancellation", func(t *testing.T) { + // Use viper to configure instead of direct field access + cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles) + + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxFiles, 10) + bm := NewBackpressureManager() + + fileCh := make(chan string, 10) + writeCh := make(chan WriteRequest, 10) + + // Fill channel + for i := 0; i < 10; i++ { + fileCh <- "file.txt" + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + // Use timeout to verify it returns quickly + done := make(chan struct{}) + go func() { + bm.WaitForChannelSpace(ctx, fileCh, writeCh) + close(done) + }() + + // Should return quickly when context is cancelled + select { + case <-done: + // Success - returned due to cancellation + case <-time.After(fastOpTimeout): + t.Fatal("WaitForChannelSpace should return immediately when context is cancelled") + } + + // Clean up + for i := 0; i < 10; i++ { + <-fileCh + } + close(fileCh) + close(writeCh) + }) +} diff --git a/fileproc/backpressure_concurrency_test.go b/fileproc/backpressure_concurrency_test.go new file mode 100644 index 0000000..ad4b897 --- /dev/null +++ b/fileproc/backpressure_concurrency_test.go @@ -0,0 +1,195 @@ +package fileproc + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackpressureManagerConcurrency(t *testing.T) { + // Configure via viper instead of direct field access + origEnabled := viper.Get(testBackpressureEnabled) + t.Cleanup(func() { + if origEnabled != nil { + viper.Set(testBackpressureEnabled, origEnabled) + } + }) + viper.Set(testBackpressureEnabled, true) + + bm := NewBackpressureManager() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + + // Multiple goroutines checking backpressure + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + bm.ShouldApplyBackpressure(ctx) + }() + } + + // Multiple goroutines applying backpressure + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + bm.ApplyBackpressure(ctx) + }() + } + + // Multiple goroutines getting stats + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + bm.GetStats() + }() + } + + // Multiple goroutines creating channels + // Note: CreateChannels returns new channels each time, caller owns them + type channelResult struct { + fileCh chan string + writeCh chan WriteRequest + } + results := make(chan channelResult, 3) + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + fileCh, writeCh := bm.CreateChannels() + results <- channelResult{fileCh, writeCh} + }() + } + + wg.Wait() + close(results) + + // Verify channels are created and have expected properties + for result := range results { + assert.NotNil(t, result.fileCh) + assert.NotNil(t, result.writeCh) + // Close channels to prevent resource leak (caller owns them) + close(result.fileCh) + close(result.writeCh) + } + + // Verify stats are consistent + stats := bm.GetStats() + assert.GreaterOrEqual(t, stats.FilesProcessed, int64(10)) +} + +func TestBackpressureManagerIntegration(t *testing.T) { + // Configure via viper instead of direct field access + origEnabled := viper.Get(testBackpressureEnabled) + origMaxFiles := viper.Get(testBackpressureMaxFiles) + origMaxWrites := viper.Get(testBackpressureMaxWrites) + origCheckInterval := viper.Get(testBackpressureMemoryCheck) + origMaxMemory := viper.Get(testBackpressureMaxMemory) + t.Cleanup(func() { + if origEnabled != nil { + viper.Set(testBackpressureEnabled, origEnabled) + } + if origMaxFiles != nil { + viper.Set(testBackpressureMaxFiles, origMaxFiles) + } + if origMaxWrites != nil { + viper.Set(testBackpressureMaxWrites, origMaxWrites) + } + if origCheckInterval != nil { + viper.Set(testBackpressureMemoryCheck, origCheckInterval) + } + if origMaxMemory != nil { + viper.Set(testBackpressureMaxMemory, origMaxMemory) + } + }) + + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxFiles, 10) + viper.Set(testBackpressureMaxWrites, 10) + viper.Set(testBackpressureMemoryCheck, 10) + viper.Set(testBackpressureMaxMemory, 100*1024*1024) // 100MB + + bm := NewBackpressureManager() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create channels - caller owns these channels and is responsible for closing them + fileCh, writeCh := bm.CreateChannels() + require.NotNil(t, fileCh) + require.NotNil(t, writeCh) + require.Greater(t, cap(fileCh), 0, "fileCh should be buffered") + require.Greater(t, cap(writeCh), 0, "writeCh should be buffered") + + // Simulate file processing + var wg sync.WaitGroup + + // Producer + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + // Check for backpressure + if bm.ShouldApplyBackpressure(ctx) { + bm.ApplyBackpressure(ctx) + } + + // Wait for channel space if needed + bm.WaitForChannelSpace(ctx, fileCh, writeCh) + + select { + case fileCh <- "file.txt": + // File sent + case <-ctx.Done(): + return + } + } + }() + + // Consumer + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + select { + case <-fileCh: + // Process file (do not manually increment filesProcessed) + case <-ctx.Done(): + return + } + } + }() + + // Wait for completion + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Success + case <-time.After(5 * time.Second): + t.Fatal("Integration test timeout") + } + + // Log final info + bm.LogBackpressureInfo() + + // Check final stats + stats := bm.GetStats() + assert.GreaterOrEqual(t, stats.FilesProcessed, int64(100)) + + // Clean up - caller owns the channels, safe to close now that goroutines have finished + close(fileCh) + close(writeCh) +} diff --git a/fileproc/backpressure_init_test.go b/fileproc/backpressure_init_test.go new file mode 100644 index 0000000..3660392 --- /dev/null +++ b/fileproc/backpressure_init_test.go @@ -0,0 +1,151 @@ +package fileproc + +import ( + "context" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +// setupViperCleanup is a test helper that captures and restores viper configuration. +// It takes a testing.T and a list of config keys to save/restore. +func setupViperCleanup(t *testing.T, keys []string) { + t.Helper() + // Capture original values and track which keys existed + origValues := make(map[string]interface{}) + keysExisted := make(map[string]bool) + for _, key := range keys { + val := viper.Get(key) + origValues[key] = val + keysExisted[key] = viper.IsSet(key) + } + // Register cleanup to restore values + t.Cleanup(func() { + for _, key := range keys { + if keysExisted[key] { + viper.Set(key, origValues[key]) + } else { + // Key didn't exist originally, so remove it + allSettings := viper.AllSettings() + delete(allSettings, key) + viper.Reset() + for k, v := range allSettings { + viper.Set(k, v) + } + } + } + }) +} + +func TestNewBackpressureManager(t *testing.T) { + keys := []string{ + testBackpressureEnabled, + testBackpressureMaxMemory, + testBackpressureMemoryCheck, + testBackpressureMaxFiles, + testBackpressureMaxWrites, + } + setupViperCleanup(t, keys) + + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxMemory, 100) + viper.Set(testBackpressureMemoryCheck, 10) + viper.Set(testBackpressureMaxFiles, 10) + viper.Set(testBackpressureMaxWrites, 10) + + bm := NewBackpressureManager() + assert.NotNil(t, bm) + assert.True(t, bm.enabled) + assert.Greater(t, bm.maxMemoryUsage, int64(0)) + assert.Greater(t, bm.memoryCheckInterval, 0) + assert.Greater(t, bm.maxPendingFiles, 0) + assert.Greater(t, bm.maxPendingWrites, 0) + assert.Equal(t, int64(0), bm.filesProcessed) +} + +func TestBackpressureStatsStructure(t *testing.T) { + // Behavioral test that exercises BackpressureManager and validates stats + keys := []string{ + testBackpressureEnabled, + testBackpressureMaxMemory, + testBackpressureMemoryCheck, + testBackpressureMaxFiles, + testBackpressureMaxWrites, + } + setupViperCleanup(t, keys) + + // Configure backpressure with realistic settings + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMaxMemory, 100*1024*1024) // 100MB + viper.Set(testBackpressureMemoryCheck, 1) // Check every file + viper.Set(testBackpressureMaxFiles, 1000) + viper.Set(testBackpressureMaxWrites, 500) + + bm := NewBackpressureManager() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Simulate processing files + initialStats := bm.GetStats() + assert.True(t, initialStats.Enabled, "backpressure should be enabled") + assert.Equal(t, int64(0), initialStats.FilesProcessed, "initially no files processed") + + // Capture initial timestamp to verify it gets updated + initialLastCheck := initialStats.LastMemoryCheck + + // Process some files to trigger memory checks + for i := 0; i < 5; i++ { + bm.ShouldApplyBackpressure(ctx) + } + + // Verify stats reflect the operations + stats := bm.GetStats() + assert.True(t, stats.Enabled, "enabled flag should be set") + assert.Equal(t, int64(5), stats.FilesProcessed, "should have processed 5 files") + assert.Greater(t, stats.CurrentMemoryUsage, int64(0), "memory usage should be tracked") + assert.Equal(t, int64(100*1024*1024), stats.MaxMemoryUsage, "max memory should match config") + assert.Equal(t, 1000, stats.MaxPendingFiles, "maxPendingFiles should match config") + assert.Equal(t, 500, stats.MaxPendingWrites, "maxPendingWrites should match config") + assert.True(t, stats.LastMemoryCheck.After(initialLastCheck) || stats.LastMemoryCheck.Equal(initialLastCheck), + "lastMemoryCheck should be updated or remain initialized") +} + +func TestBackpressureManagerGetStats(t *testing.T) { + keys := []string{ + testBackpressureEnabled, + testBackpressureMemoryCheck, + } + setupViperCleanup(t, keys) + + // Ensure config enables backpressure and checks every call + viper.Set(testBackpressureEnabled, true) + viper.Set(testBackpressureMemoryCheck, 1) + + bm := NewBackpressureManager() + + // Capture initial timestamp to verify it gets updated + initialStats := bm.GetStats() + initialLastCheck := initialStats.LastMemoryCheck + + // Process some files to update stats + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for i := 0; i < 5; i++ { + bm.ShouldApplyBackpressure(ctx) + } + + stats := bm.GetStats() + + assert.True(t, stats.Enabled) + assert.Equal(t, int64(5), stats.FilesProcessed) + assert.Greater(t, stats.CurrentMemoryUsage, int64(0)) + assert.Equal(t, bm.maxMemoryUsage, stats.MaxMemoryUsage) + assert.Equal(t, bm.maxPendingFiles, stats.MaxPendingFiles) + assert.Equal(t, bm.maxPendingWrites, stats.MaxPendingWrites) + + // LastMemoryCheck should be updated after processing files (memoryCheckInterval=1) + assert.True(t, stats.LastMemoryCheck.After(initialLastCheck), + "lastMemoryCheck should be updated after memory checks") +} diff --git a/fileproc/config.go b/fileproc/config.go index 24d59e0..06f13e6 100644 --- a/fileproc/config.go +++ b/fileproc/config.go @@ -1,9 +1,162 @@ package fileproc -import "strings" +import ( + "fmt" + "path/filepath" + "strings" +) + +const ( + // MaxRegistryEntries is the maximum number of entries allowed in registry config slices/maps. + MaxRegistryEntries = 1000 + // MaxExtensionLength is the maximum length for a single extension string. + MaxExtensionLength = 100 +) + +// RegistryConfig holds configuration for file type registry. +// All paths must be relative without path traversal (no ".." or leading "/"). +// Extensions in CustomLanguages keys must start with "." or be alphanumeric with underscore/hyphen. +type RegistryConfig struct { + // CustomImages: file extensions to treat as images (e.g., ".svg", ".webp"). + // Must be relative paths without ".." or leading separators. + CustomImages []string + + // CustomBinary: file extensions to treat as binary (e.g., ".bin", ".dat"). + // Must be relative paths without ".." or leading separators. + CustomBinary []string + + // CustomLanguages: maps file extensions to language names (e.g., {".tsx": "TypeScript"}). + // Keys must start with "." or be alphanumeric with underscore/hyphen. + CustomLanguages map[string]string + + // DisabledImages: image extensions to disable from default registry. + DisabledImages []string + + // DisabledBinary: binary extensions to disable from default registry. + DisabledBinary []string + + // DisabledLanguages: language extensions to disable from default registry. + DisabledLanguages []string +} + +// Validate checks the RegistryConfig for invalid entries and enforces limits. +func (c *RegistryConfig) Validate() error { + // Validate CustomImages + if err := validateExtensionSlice(c.CustomImages, "CustomImages"); err != nil { + return err + } + + // Validate CustomBinary + if err := validateExtensionSlice(c.CustomBinary, "CustomBinary"); err != nil { + return err + } + + // Validate CustomLanguages + if len(c.CustomLanguages) > MaxRegistryEntries { + return fmt.Errorf( + "CustomLanguages exceeds maximum entries (%d > %d)", + len(c.CustomLanguages), + MaxRegistryEntries, + ) + } + for ext, lang := range c.CustomLanguages { + if err := validateExtension(ext, "CustomLanguages key"); err != nil { + return err + } + if len(lang) > MaxExtensionLength { + return fmt.Errorf( + "CustomLanguages value %q exceeds maximum length (%d > %d)", + lang, + len(lang), + MaxExtensionLength, + ) + } + } + + // Validate Disabled slices + if err := validateExtensionSlice(c.DisabledImages, "DisabledImages"); err != nil { + return err + } + if err := validateExtensionSlice(c.DisabledBinary, "DisabledBinary"); err != nil { + return err + } + + return validateExtensionSlice(c.DisabledLanguages, "DisabledLanguages") +} + +// validateExtensionSlice validates a slice of extensions for path safety and limits. +func validateExtensionSlice(slice []string, fieldName string) error { + if len(slice) > MaxRegistryEntries { + return fmt.Errorf("%s exceeds maximum entries (%d > %d)", fieldName, len(slice), MaxRegistryEntries) + } + for _, ext := range slice { + if err := validateExtension(ext, fieldName); err != nil { + return err + } + } + return nil +} + +// validateExtension validates a single extension for path safety. +// +//revive:disable-next-line:cyclomatic +func validateExtension(ext, context string) error { + // Reject empty strings + if ext == "" { + return fmt.Errorf("%s entry cannot be empty", context) + } + + if len(ext) > MaxExtensionLength { + return fmt.Errorf( + "%s entry %q exceeds maximum length (%d > %d)", + context, ext, len(ext), MaxExtensionLength, + ) + } + + // Reject absolute paths + if filepath.IsAbs(ext) { + return fmt.Errorf("%s entry %q is an absolute path (not allowed)", context, ext) + } + + // Reject path traversal + if strings.Contains(ext, "..") { + return fmt.Errorf("%s entry %q contains path traversal (not allowed)", context, ext) + } + + // For extensions, verify they start with "." or are alphanumeric + if strings.HasPrefix(ext, ".") { + // Reject extensions containing path separators + if strings.ContainsRune(ext, filepath.Separator) || strings.ContainsRune(ext, '/') || + strings.ContainsRune(ext, '\\') { + return fmt.Errorf("%s entry %q contains path separators (not allowed)", context, ext) + } + // Valid extension format + return nil + } + + // Check if purely alphanumeric (for bare names) + for _, r := range ext { + isValid := (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || + r == '_' || r == '-' + if !isValid { + return fmt.Errorf( + "%s entry %q contains invalid characters (must start with '.' or be alphanumeric/_/-)", + context, + ext, + ) + } + } + + return nil +} // ApplyCustomExtensions applies custom extensions from configuration. -func (r *FileTypeRegistry) ApplyCustomExtensions(customImages, customBinary []string, customLanguages map[string]string) { +func (r *FileTypeRegistry) ApplyCustomExtensions( + customImages, customBinary []string, + customLanguages map[string]string, +) { // Add custom image extensions r.addExtensions(customImages, r.AddImageExtension) @@ -29,12 +182,24 @@ func (r *FileTypeRegistry) addExtensions(extensions []string, adder func(string) // ConfigureFromSettings applies configuration settings to the registry. // This function is called from main.go after config is loaded to avoid circular imports. -func ConfigureFromSettings( - customImages, customBinary []string, - customLanguages map[string]string, - disabledImages, disabledBinary, disabledLanguages []string, -) { +// It validates the configuration before applying it. +func ConfigureFromSettings(config RegistryConfig) error { + // Validate configuration first + if err := config.Validate(); err != nil { + return err + } + registry := GetDefaultRegistry() - registry.ApplyCustomExtensions(customImages, customBinary, customLanguages) - registry.DisableExtensions(disabledImages, disabledBinary, disabledLanguages) + + // Only apply custom extensions if they are non-empty (len() for nil slices/maps is zero) + if len(config.CustomImages) > 0 || len(config.CustomBinary) > 0 || len(config.CustomLanguages) > 0 { + registry.ApplyCustomExtensions(config.CustomImages, config.CustomBinary, config.CustomLanguages) + } + + // Only disable extensions if they are non-empty + if len(config.DisabledImages) > 0 || len(config.DisabledBinary) > 0 || len(config.DisabledLanguages) > 0 { + registry.DisableExtensions(config.DisabledImages, config.DisabledBinary, config.DisabledLanguages) + } + + return nil } diff --git a/fileproc/filetypes_concurrency_test.go b/fileproc/filetypes_concurrency_test.go index 9478aac..817d4b6 100644 --- a/fileproc/filetypes_concurrency_test.go +++ b/fileproc/filetypes_concurrency_test.go @@ -14,10 +14,10 @@ func TestFileTypeRegistry_ThreadSafety(t *testing.T) { var wg sync.WaitGroup // Test concurrent read operations - t.Run("ConcurrentReads", func(t *testing.T) { + t.Run("ConcurrentReads", func(_ *testing.T) { for i := 0; i < numGoroutines; i++ { wg.Add(1) - go func(id int) { + go func(_ int) { defer wg.Done() registry := GetDefaultRegistry() @@ -102,4 +102,4 @@ func TestFileTypeRegistry_ThreadSafety(t *testing.T) { } wg.Wait() }) -} \ No newline at end of file +} diff --git a/fileproc/filetypes_config_test.go b/fileproc/filetypes_config_test.go index 9690a7f..df1e8c5 100644 --- a/fileproc/filetypes_config_test.go +++ b/fileproc/filetypes_config_test.go @@ -1,8 +1,9 @@ package fileproc import ( - "sync" "testing" + + "github.com/stretchr/testify/require" ) // TestFileTypeRegistry_Configuration tests the configuration functionality. @@ -142,7 +143,7 @@ func TestFileTypeRegistry_Configuration(t *testing.T) { } }) - // Test case insensitive handling + // Test case-insensitive handling t.Run("CaseInsensitiveHandling", func(t *testing.T) { registry := &FileTypeRegistry{ imageExts: make(map[string]bool), @@ -184,8 +185,9 @@ func TestFileTypeRegistry_Configuration(t *testing.T) { // TestConfigureFromSettings tests the global configuration function. func TestConfigureFromSettings(t *testing.T) { // Reset registry to ensure clean state - registryOnce = sync.Once{} - registry = nil + ResetRegistryForTesting() + // Ensure cleanup runs even if test fails + t.Cleanup(ResetRegistryForTesting) // Test configuration application customImages := []string{".webp", ".avif"} @@ -195,14 +197,15 @@ func TestConfigureFromSettings(t *testing.T) { disabledBinary := []string{".exe"} // Disable default extension disabledLanguages := []string{".rb"} // Disable default extension - ConfigureFromSettings( - customImages, - customBinary, - customLanguages, - disabledImages, - disabledBinary, - disabledLanguages, - ) + err := ConfigureFromSettings(RegistryConfig{ + CustomImages: customImages, + CustomBinary: customBinary, + CustomLanguages: customLanguages, + DisabledImages: disabledImages, + DisabledBinary: disabledBinary, + DisabledLanguages: disabledLanguages, + }) + require.NoError(t, err) // Test that custom extensions work if !IsImage("test.webp") { @@ -238,14 +241,15 @@ func TestConfigureFromSettings(t *testing.T) { } // Test multiple calls don't override previous configuration - ConfigureFromSettings( - []string{".extra"}, - []string{}, - map[string]string{}, - []string{}, - []string{}, - []string{}, - ) + err = ConfigureFromSettings(RegistryConfig{ + CustomImages: []string{".extra"}, + CustomBinary: []string{}, + CustomLanguages: map[string]string{}, + DisabledImages: []string{}, + DisabledBinary: []string{}, + DisabledLanguages: []string{}, + }) + require.NoError(t, err) // Previous configuration should still work if !IsImage("test.webp") { @@ -255,4 +259,4 @@ func TestConfigureFromSettings(t *testing.T) { if !IsImage("test.extra") { t.Error("Expected new configuration to be applied") } -} \ No newline at end of file +} diff --git a/fileproc/filetypes_detection_test.go b/fileproc/filetypes_detection_test.go index d3a9acd..5364367 100644 --- a/fileproc/filetypes_detection_test.go +++ b/fileproc/filetypes_detection_test.go @@ -4,9 +4,21 @@ import ( "testing" ) +// newTestRegistry creates a fresh registry instance for testing to avoid global state pollution. +func newTestRegistry() *FileTypeRegistry { + return &FileTypeRegistry{ + imageExts: getImageExtensions(), + binaryExts: getBinaryExtensions(), + languageMap: getLanguageMap(), + extCache: make(map[string]string, 1000), + resultCache: make(map[string]FileTypeResult, 500), + maxCacheSize: 500, + } +} + // TestFileTypeRegistry_LanguageDetection tests the language detection functionality. func TestFileTypeRegistry_LanguageDetection(t *testing.T) { - registry := GetDefaultRegistry() + registry := newTestRegistry() tests := []struct { filename string @@ -94,7 +106,7 @@ func TestFileTypeRegistry_LanguageDetection(t *testing.T) { // TestFileTypeRegistry_ImageDetection tests the image detection functionality. func TestFileTypeRegistry_ImageDetection(t *testing.T) { - registry := GetDefaultRegistry() + registry := newTestRegistry() tests := []struct { filename string @@ -144,7 +156,7 @@ func TestFileTypeRegistry_ImageDetection(t *testing.T) { // TestFileTypeRegistry_BinaryDetection tests the binary detection functionality. func TestFileTypeRegistry_BinaryDetection(t *testing.T) { - registry := GetDefaultRegistry() + registry := newTestRegistry() tests := []struct { filename string @@ -208,11 +220,11 @@ func TestFileTypeRegistry_BinaryDetection(t *testing.T) { {"page.html", false}, // Edge cases - {"", false}, // Empty filename - {"binary", false}, // No extension - {".exe", true}, // Just extension - {"file.exe.txt", false}, // Multiple extensions - {"file.unknown", false}, // Unknown extension + {"", false}, // Empty filename + {"binary", false}, // No extension + {".exe", true}, // Just extension + {"file.exe.txt", false}, // Multiple extensions + {"file.unknown", false}, // Unknown extension } for _, tt := range tests { @@ -223,4 +235,4 @@ func TestFileTypeRegistry_BinaryDetection(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/fileproc/filetypes_edge_cases_test.go b/fileproc/filetypes_edge_cases_test.go index ce9ee84..10eb083 100644 --- a/fileproc/filetypes_edge_cases_test.go +++ b/fileproc/filetypes_edge_cases_test.go @@ -31,7 +31,7 @@ func TestFileTypeRegistry_EdgeCases(t *testing.T) { } for _, tc := range edgeCases { - t.Run(tc.name, func(t *testing.T) { + t.Run(tc.name, func(_ *testing.T) { // These should not panic _ = registry.IsImage(tc.filename) _ = registry.IsBinary(tc.filename) @@ -125,4 +125,4 @@ func BenchmarkFileTypeRegistry_ConcurrentAccess(b *testing.B) { _ = GetLanguage(filename) } }) -} \ No newline at end of file +} diff --git a/fileproc/filetypes_registry_test.go b/fileproc/filetypes_registry_test.go index 0b9954a..57b80fb 100644 --- a/fileproc/filetypes_registry_test.go +++ b/fileproc/filetypes_registry_test.go @@ -21,7 +21,7 @@ func TestFileTypeRegistry_ModificationMethods(t *testing.T) { t.Errorf("Expected .webp to be recognized as image after adding") } - // Test case insensitive addition + // Test case-insensitive addition registry.AddImageExtension(".AVIF") if !registry.IsImage("test.avif") { t.Errorf("Expected .avif to be recognized as image after adding .AVIF") @@ -51,7 +51,7 @@ func TestFileTypeRegistry_ModificationMethods(t *testing.T) { t.Errorf("Expected .custom to be recognized as binary after adding") } - // Test case insensitive addition + // Test case-insensitive addition registry.AddBinaryExtension(".SPECIAL") if !registry.IsBinary("file.special") { t.Errorf("Expected .special to be recognized as binary after adding .SPECIAL") @@ -81,7 +81,7 @@ func TestFileTypeRegistry_ModificationMethods(t *testing.T) { t.Errorf("Expected CustomLang, got %s", lang) } - // Test case insensitive addition + // Test case-insensitive addition registry.AddLanguageMapping(".ABC", "UpperLang") if lang := registry.GetLanguage("file.abc"); lang != "UpperLang" { t.Errorf("Expected UpperLang, got %s", lang) @@ -134,4 +134,4 @@ func TestFileTypeRegistry_DefaultRegistryConsistency(t *testing.T) { t.Errorf("Iteration %d: Expected .txt to not be recognized as binary", i) } } -} \ No newline at end of file +} diff --git a/fileproc/json_writer.go b/fileproc/json_writer.go index 5c9ab46..59abb2b 100644 --- a/fileproc/json_writer.go +++ b/fileproc/json_writer.go @@ -6,7 +6,7 @@ import ( "io" "os" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // JSONWriter handles JSON format output with streaming support. @@ -27,27 +27,42 @@ func NewJSONWriter(outFile *os.File) *JSONWriter { func (w *JSONWriter) Start(prefix, suffix string) error { // Start JSON structure if _, err := w.outFile.WriteString(`{"prefix":"`); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON start") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write JSON start", + ) } // Write escaped prefix - escapedPrefix := utils.EscapeForJSON(prefix) - if err := utils.WriteWithErrorWrap(w.outFile, escapedPrefix, "failed to write JSON prefix", ""); err != nil { + escapedPrefix := gibidiutils.EscapeForJSON(prefix) + if err := gibidiutils.WriteWithErrorWrap(w.outFile, escapedPrefix, "failed to write JSON prefix", ""); err != nil { return err } if _, err := w.outFile.WriteString(`","suffix":"`); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON middle") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write JSON middle", + ) } // Write escaped suffix - escapedSuffix := utils.EscapeForJSON(suffix) - if err := utils.WriteWithErrorWrap(w.outFile, escapedSuffix, "failed to write JSON suffix", ""); err != nil { + escapedSuffix := gibidiutils.EscapeForJSON(suffix) + if err := gibidiutils.WriteWithErrorWrap(w.outFile, escapedSuffix, "failed to write JSON suffix", ""); err != nil { return err } if _, err := w.outFile.WriteString(`","files":[`); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON files start") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write JSON files start", + ) } return nil @@ -57,7 +72,12 @@ func (w *JSONWriter) Start(prefix, suffix string) error { func (w *JSONWriter) WriteFile(req WriteRequest) error { if !w.firstFile { if _, err := w.outFile.WriteString(","); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON separator") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write JSON separator", + ) } } w.firstFile = false @@ -72,21 +92,24 @@ func (w *JSONWriter) WriteFile(req WriteRequest) error { func (w *JSONWriter) Close() error { // Close JSON structure if _, err := w.outFile.WriteString("]}"); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON end") + return gibidiutils.WrapError(err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, "failed to write JSON end") } return nil } // writeStreaming writes a large file as JSON in streaming chunks. func (w *JSONWriter) writeStreaming(req WriteRequest) error { - defer utils.SafeCloseReader(req.Reader, req.Path) + defer gibidiutils.SafeCloseReader(req.Reader, req.Path) language := detectLanguage(req.Path) // Write file start - escapedPath := utils.EscapeForJSON(req.Path) + escapedPath := gibidiutils.EscapeForJSON(req.Path) if _, err := fmt.Fprintf(w.outFile, `{"path":"%s","language":"%s","content":"`, escapedPath, language); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON file start").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write JSON file start", + ).WithFilePath(req.Path) } // Stream content with JSON escaping @@ -96,7 +119,10 @@ func (w *JSONWriter) writeStreaming(req WriteRequest) error { // Write file end if _, err := w.outFile.WriteString(`"}`); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON file end").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write JSON file end", + ).WithFilePath(req.Path) } return nil @@ -113,25 +139,29 @@ func (w *JSONWriter) writeInline(req WriteRequest) error { encoded, err := json.Marshal(fileData) if err != nil { - return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingEncode, "failed to marshal JSON").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingEncode, + "failed to marshal JSON", + ).WithFilePath(req.Path) } if _, err := w.outFile.Write(encoded); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON file").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write JSON file", + ).WithFilePath(req.Path) } return nil } // streamJSONContent streams content with JSON escaping. func (w *JSONWriter) streamJSONContent(reader io.Reader, path string) error { - return utils.StreamContent(reader, w.outFile, StreamChunkSize, path, func(chunk []byte) []byte { - escaped := utils.EscapeForJSON(string(chunk)) + return gibidiutils.StreamContent(reader, w.outFile, StreamChunkSize, path, func(chunk []byte) []byte { + escaped := gibidiutils.EscapeForJSON(string(chunk)) return []byte(escaped) }) } - - // startJSONWriter handles JSON format output with streaming support. func startJSONWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) { defer close(done) @@ -140,19 +170,19 @@ func startJSONWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- // Start writing if err := writer.Start(prefix, suffix); err != nil { - utils.LogError("Failed to write JSON start", err) + gibidiutils.LogError("Failed to write JSON start", err) return } // Process files for req := range writeCh { if err := writer.WriteFile(req); err != nil { - utils.LogError("Failed to write JSON file", err) + gibidiutils.LogError("Failed to write JSON file", err) } } // Close writer if err := writer.Close(); err != nil { - utils.LogError("Failed to write JSON end", err) + gibidiutils.LogError("Failed to write JSON end", err) } } diff --git a/fileproc/markdown_writer.go b/fileproc/markdown_writer.go index 56e5fdf..0f5ebd2 100644 --- a/fileproc/markdown_writer.go +++ b/fileproc/markdown_writer.go @@ -4,11 +4,13 @@ import ( "fmt" "io" "os" + "path/filepath" + "strings" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) -// MarkdownWriter handles markdown format output with streaming support. +// MarkdownWriter handles Markdown format output with streaming support. type MarkdownWriter struct { outFile *os.File } @@ -19,16 +21,21 @@ func NewMarkdownWriter(outFile *os.File) *MarkdownWriter { } // Start writes the markdown header. -func (w *MarkdownWriter) Start(prefix, suffix string) error { +func (w *MarkdownWriter) Start(prefix, _ string) error { if prefix != "" { if _, err := fmt.Fprintf(w.outFile, "# %s\n\n", prefix); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write prefix") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write prefix", + ) } } return nil } -// WriteFile writes a file entry in markdown format. +// WriteFile writes a file entry in Markdown format. func (w *MarkdownWriter) WriteFile(req WriteRequest) error { if req.IsStream { return w.writeStreaming(req) @@ -40,21 +47,99 @@ func (w *MarkdownWriter) WriteFile(req WriteRequest) error { func (w *MarkdownWriter) Close(suffix string) error { if suffix != "" { if _, err := fmt.Fprintf(w.outFile, "\n# %s\n", suffix); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write suffix") + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write suffix", + ) } } return nil } +// validateMarkdownPath validates a file path for markdown output. +func validateMarkdownPath(path string) error { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationRequired, + "file path cannot be empty", + "", + nil, + ) + } + + // Reject absolute paths + if filepath.IsAbs(trimmed) { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "absolute paths are not allowed", + trimmed, + map[string]any{"path": trimmed}, + ) + } + + // Clean and validate path components + cleaned := filepath.Clean(trimmed) + if filepath.IsAbs(cleaned) || strings.HasPrefix(cleaned, "/") { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path must be relative", + trimmed, + map[string]any{"path": trimmed, "cleaned": cleaned}, + ) + } + + // Check for path traversal in components + components := strings.Split(filepath.ToSlash(cleaned), "/") + for _, component := range components { + if component == ".." { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path traversal not allowed", + trimmed, + map[string]any{"path": trimmed, "cleaned": cleaned}, + ) + } + } + + return nil +} + // writeStreaming writes a large file in streaming chunks. func (w *MarkdownWriter) writeStreaming(req WriteRequest) error { - defer w.closeReader(req.Reader, req.Path) + // Validate path before use + if err := validateMarkdownPath(req.Path); err != nil { + return err + } + + // Check for nil reader + if req.Reader == nil { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationRequired, + "nil reader in write request", + "", + nil, + ).WithFilePath(req.Path) + } + + defer gibidiutils.SafeCloseReader(req.Reader, req.Path) language := detectLanguage(req.Path) // Write file header - if _, err := fmt.Fprintf(w.outFile, "## File: `%s`\n```%s\n", req.Path, language); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write file header").WithFilePath(req.Path) + safePath := gibidiutils.EscapeForMarkdown(req.Path) + if _, err := fmt.Fprintf(w.outFile, "## File: `%s`\n```%s\n", safePath, language); err != nil { + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write file header", + ).WithFilePath(req.Path) } // Stream file content in chunks @@ -64,7 +149,10 @@ func (w *MarkdownWriter) writeStreaming(req WriteRequest) error { // Write file footer if _, err := w.outFile.WriteString("\n```\n\n"); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write file footer").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write file footer", + ).WithFilePath(req.Path) } return nil @@ -72,68 +160,55 @@ func (w *MarkdownWriter) writeStreaming(req WriteRequest) error { // writeInline writes a small file directly from content. func (w *MarkdownWriter) writeInline(req WriteRequest) error { + // Validate path before use + if err := validateMarkdownPath(req.Path); err != nil { + return err + } + language := detectLanguage(req.Path) - formatted := fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", req.Path, language, req.Content) + safePath := gibidiutils.EscapeForMarkdown(req.Path) + formatted := fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", safePath, language, req.Content) if _, err := w.outFile.WriteString(formatted); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write inline content").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write inline content", + ).WithFilePath(req.Path) } return nil } // streamContent streams file content in chunks. func (w *MarkdownWriter) streamContent(reader io.Reader, path string) error { - buf := make([]byte, StreamChunkSize) - for { - n, err := reader.Read(buf) - if n > 0 { - if _, writeErr := w.outFile.Write(buf[:n]); writeErr != nil { - return utils.WrapError(writeErr, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write chunk").WithFilePath(path) - } - } - if err == io.EOF { - break - } - if err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIORead, "failed to read chunk").WithFilePath(path) - } - } - return nil + return gibidiutils.StreamContent(reader, w.outFile, StreamChunkSize, path, nil) } -// closeReader safely closes a reader if it implements io.Closer. -func (w *MarkdownWriter) closeReader(reader io.Reader, path string) { - if closer, ok := reader.(io.Closer); ok { - if err := closer.Close(); err != nil { - utils.LogError( - "Failed to close file reader", - utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOClose, "failed to close file reader").WithFilePath(path), - ) - } - } -} - -// startMarkdownWriter handles markdown format output with streaming support. -func startMarkdownWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) { +// startMarkdownWriter handles Markdown format output with streaming support. +func startMarkdownWriter( + outFile *os.File, + writeCh <-chan WriteRequest, + done chan<- struct{}, + prefix, suffix string, +) { defer close(done) writer := NewMarkdownWriter(outFile) // Start writing if err := writer.Start(prefix, suffix); err != nil { - utils.LogError("Failed to write markdown prefix", err) + gibidiutils.LogError("Failed to write markdown prefix", err) return } // Process files for req := range writeCh { if err := writer.WriteFile(req); err != nil { - utils.LogError("Failed to write markdown file", err) + gibidiutils.LogError("Failed to write markdown file", err) } } // Close writer if err := writer.Close(suffix); err != nil { - utils.LogError("Failed to write markdown suffix", err) + gibidiutils.LogError("Failed to write markdown suffix", err) } } diff --git a/fileproc/processor.go b/fileproc/processor.go index f9ff983..bfa7e8d 100644 --- a/fileproc/processor.go +++ b/fileproc/processor.go @@ -3,6 +3,7 @@ package fileproc import ( "context" + "errors" "fmt" "io" "os" @@ -13,7 +14,7 @@ import ( "github.com/sirupsen/logrus" "github.com/ivuorinen/gibidify/config" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) const ( @@ -33,6 +34,26 @@ type WriteRequest struct { Reader io.Reader } +// multiReaderCloser wraps an io.Reader with a Close method that closes underlying closers. +type multiReaderCloser struct { + reader io.Reader + closers []io.Closer +} + +func (m *multiReaderCloser) Read(p []byte) (n int, err error) { + return m.reader.Read(p) +} + +func (m *multiReaderCloser) Close() error { + var firstErr error + for _, c := range m.closers { + if err := c.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + // FileProcessor handles file processing operations. type FileProcessor struct { rootPath string @@ -58,6 +79,34 @@ func NewFileProcessorWithMonitor(rootPath string, monitor *ResourceMonitor) *Fil } } +// checkContextCancellation checks if context is cancelled and logs an error if so. +// Returns true if context is cancelled, false otherwise. +func (p *FileProcessor) checkContextCancellation(ctx context.Context, filePath, stage string) bool { + select { + case <-ctx.Done(): + // Format stage with leading space if provided + stageMsg := stage + if stage != "" { + stageMsg = " " + stage + } + gibidiutils.LogErrorf( + gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitTimeout, + fmt.Sprintf("file processing cancelled%s", stageMsg), + filePath, + nil, + ), + "File processing cancelled%s: %s", + stageMsg, + filePath, + ) + return true + default: + return false + } +} + // ProcessFile reads the file at filePath and sends a formatted output to outCh. // It automatically chooses between loading the entire file or streaming based on file size. func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) { @@ -67,7 +116,13 @@ func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) { } // ProcessFileWithMonitor processes a file using a shared resource monitor. -func ProcessFileWithMonitor(ctx context.Context, filePath string, outCh chan<- WriteRequest, rootPath string, monitor *ResourceMonitor) { +func ProcessFileWithMonitor( + ctx context.Context, + filePath string, + outCh chan<- WriteRequest, + rootPath string, + monitor *ResourceMonitor, +) { processor := NewFileProcessorWithMonitor(rootPath, monitor) processor.ProcessWithContext(ctx, filePath, outCh) } @@ -86,10 +141,17 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, // Wait for rate limiting if err := p.resourceMonitor.WaitForRateLimit(fileCtx); err != nil { - if err == context.DeadlineExceeded { - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing timeout during rate limiting", filePath, nil), - "File processing timeout during rate limiting: %s", filePath, + if errors.Is(err, context.DeadlineExceeded) { + gibidiutils.LogErrorf( + gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitTimeout, + "file processing timeout during rate limiting", + filePath, + nil, + ), + "File processing timeout during rate limiting: %s", + filePath, ) } return @@ -103,10 +165,17 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, // Acquire read slot for concurrent processing if err := p.resourceMonitor.AcquireReadSlot(fileCtx); err != nil { - if err == context.DeadlineExceeded { - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing timeout waiting for read slot", filePath, nil), - "File processing timeout waiting for read slot: %s", filePath, + if errors.Is(err, context.DeadlineExceeded) { + gibidiutils.LogErrorf( + gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitTimeout, + "file processing timeout waiting for read slot", + filePath, + nil, + ), + "File processing timeout waiting for read slot: %s", + filePath, ) } return @@ -115,7 +184,7 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, // Check hard memory limits before processing if err := p.resourceMonitor.CheckHardMemoryLimit(); err != nil { - utils.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath) + gibidiutils.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath) return } @@ -138,7 +207,6 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string, } } - // validateFileWithLimits checks if the file can be processed with resource limits. func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath string) (os.FileInfo, error) { // Check context cancellation @@ -150,24 +218,27 @@ func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath str fileInfo, err := os.Stat(filePath) if err != nil { - structErr := utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to stat file").WithFilePath(filePath) - utils.LogErrorf(structErr, "Failed to stat file %s", filePath) - return nil, err + structErr := gibidiutils.WrapError( + err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSAccess, + "failed to stat file", + ).WithFilePath(filePath) + gibidiutils.LogErrorf(structErr, "Failed to stat file %s", filePath) + return nil, structErr } // Check traditional size limit if fileInfo.Size() > p.sizeLimit { - context := map[string]interface{}{ + filesizeContext := map[string]interface{}{ "file_size": fileInfo.Size(), "size_limit": p.sizeLimit, } - utils.LogErrorf( - utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeValidationSize, + gibidiutils.LogErrorf( + gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationSize, fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", fileInfo.Size(), p.sizeLimit), filePath, - context, + filesizeContext, ), "Skipping large file %s", filePath, ) @@ -176,7 +247,7 @@ func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath str // Check resource limits if err := p.resourceMonitor.ValidateFileProcessing(filePath, fileInfo.Size()); err != nil { - utils.LogErrorf(err, "Resource limit validation failed for file: %s", filePath) + gibidiutils.LogErrorf(err, "Resource limit validation failed for file: %s", filePath) return nil, err } @@ -192,66 +263,54 @@ func (p *FileProcessor) getRelativePath(filePath string) string { return relPath } - // processInMemoryWithContext loads the entire file into memory with context awareness. -func (p *FileProcessor) processInMemoryWithContext(ctx context.Context, filePath, relPath string, outCh chan<- WriteRequest) { +func (p *FileProcessor) processInMemoryWithContext( + ctx context.Context, + filePath, relPath string, + outCh chan<- WriteRequest, +) { // Check context before reading - select { - case <-ctx.Done(): - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled", filePath, nil), - "File processing cancelled: %s", filePath, - ) + if p.checkContextCancellation(ctx, filePath, "") { return - default: } - content, err := os.ReadFile(filePath) // #nosec G304 - filePath is validated by walker + // #nosec G304 - filePath is validated by walker + content, err := os.ReadFile(filePath) if err != nil { - structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to read file").WithFilePath(filePath) - utils.LogErrorf(structErr, "Failed to read file %s", filePath) + structErr := gibidiutils.WrapError( + err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingFileRead, + "failed to read file", + ).WithFilePath(filePath) + gibidiutils.LogErrorf(structErr, "Failed to read file %s", filePath) return } // Check context again after reading - select { - case <-ctx.Done(): - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled after read", filePath, nil), - "File processing cancelled after read: %s", filePath, - ) + if p.checkContextCancellation(ctx, filePath, "after read") { return - default: } - // Try to send the result, but respect context cancellation - select { - case <-ctx.Done(): - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled before output", filePath, nil), - "File processing cancelled before output: %s", filePath, - ) + // Check context before sending output + if p.checkContextCancellation(ctx, filePath, "before output") { return - case outCh <- WriteRequest{ + } + + outCh <- WriteRequest{ Path: relPath, Content: p.formatContent(relPath, string(content)), IsStream: false, - }: } } - // processStreamingWithContext creates a streaming reader for large files with context awareness. -func (p *FileProcessor) processStreamingWithContext(ctx context.Context, filePath, relPath string, outCh chan<- WriteRequest) { +func (p *FileProcessor) processStreamingWithContext( + ctx context.Context, + filePath, relPath string, + outCh chan<- WriteRequest, +) { // Check context before creating reader - select { - case <-ctx.Done(): - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "streaming processing cancelled", filePath, nil), - "Streaming processing cancelled: %s", filePath, - ) + if p.checkContextCancellation(ctx, filePath, "before streaming") { return - default: } reader := p.createStreamReaderWithContext(ctx, filePath, relPath) @@ -259,43 +318,47 @@ func (p *FileProcessor) processStreamingWithContext(ctx context.Context, filePat return // Error already logged } - // Try to send the result, but respect context cancellation - select { - case <-ctx.Done(): - utils.LogErrorf( - utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "streaming processing cancelled before output", filePath, nil), - "Streaming processing cancelled before output: %s", filePath, - ) + // Check context before sending output + if p.checkContextCancellation(ctx, filePath, "before streaming output") { + // Close the reader to prevent file descriptor leak + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } return - case outCh <- WriteRequest{ + } + + outCh <- WriteRequest{ Path: relPath, Content: "", // Empty since content is in Reader IsStream: true, Reader: reader, - }: } } - // createStreamReaderWithContext creates a reader that combines header and file content with context awareness. func (p *FileProcessor) createStreamReaderWithContext(ctx context.Context, filePath, relPath string) io.Reader { // Check context before opening file - select { - case <-ctx.Done(): + if p.checkContextCancellation(ctx, filePath, "before opening file") { return nil - default: } - file, err := os.Open(filePath) // #nosec G304 - filePath is validated by walker + // #nosec G304 - filePath is validated by walker + file, err := os.Open(filePath) if err != nil { - structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to open file for streaming").WithFilePath(filePath) - utils.LogErrorf(structErr, "Failed to open file for streaming %s", filePath) + structErr := gibidiutils.WrapError( + err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingFileRead, + "failed to open file for streaming", + ).WithFilePath(filePath) + gibidiutils.LogErrorf(structErr, "Failed to open file for streaming %s", filePath) return nil } - // Note: file will be closed by the writer header := p.formatHeader(relPath) - return io.MultiReader(header, file) + // Wrap in multiReaderCloser to ensure file is closed even on cancellation + return &multiReaderCloser{ + reader: io.MultiReader(header, file), + closers: []io.Closer{file}, + } } // formatContent formats the file content with header. diff --git a/fileproc/resource_monitor_concurrency.go b/fileproc/resource_monitor_concurrency.go index 4d1789b..a66cc4d 100644 --- a/fileproc/resource_monitor_concurrency.go +++ b/fileproc/resource_monitor_concurrency.go @@ -51,9 +51,11 @@ func (rm *ResourceMonitor) CreateFileProcessingContext(parent context.Context) ( } // CreateOverallProcessingContext creates a context with overall processing timeout. -func (rm *ResourceMonitor) CreateOverallProcessingContext(parent context.Context) (context.Context, context.CancelFunc) { +func (rm *ResourceMonitor) CreateOverallProcessingContext( + parent context.Context, +) (context.Context, context.CancelFunc) { if !rm.enabled || rm.overallTimeout <= 0 { return parent, func() {} } return context.WithTimeout(parent, rm.overallTimeout) -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_concurrency_test.go b/fileproc/resource_monitor_concurrency_test.go index 566b037..7a886b9 100644 --- a/fileproc/resource_monitor_concurrency_test.go +++ b/fileproc/resource_monitor_concurrency_test.go @@ -35,7 +35,7 @@ func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) { t.Errorf("Expected no error for second read slot, got %v", err) } - // Third read slot should timeout (context deadline exceeded) + // Third read slot should time out (context deadline exceeded) err = rm.AcquireReadSlot(ctx) if err == nil { t.Error("Expected timeout error for third read slot, got nil") @@ -43,11 +43,11 @@ func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) { // Release one slot and try again rm.ReleaseReadSlot() - + // Create new context for the next attempt ctx2, cancel2 := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel2() - + err = rm.AcquireReadSlot(ctx2) if err != nil { t.Errorf("Expected no error after releasing a slot, got %v", err) @@ -92,4 +92,4 @@ func TestResourceMonitor_TimeoutContexts(t *testing.T) { } else if time.Until(deadline) > 2*time.Second+100*time.Millisecond { t.Error("Overall processing timeout appears to be too long") } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_integration_test.go b/fileproc/resource_monitor_integration_test.go index 4dd2915..eba2bd3 100644 --- a/fileproc/resource_monitor_integration_test.go +++ b/fileproc/resource_monitor_integration_test.go @@ -78,4 +78,4 @@ func TestResourceMonitor_Integration(t *testing.T) { // Test resource limit logging rm.LogResourceInfo() -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_metrics.go b/fileproc/resource_monitor_metrics.go index 75e767c..8bfba34 100644 --- a/fileproc/resource_monitor_metrics.go +++ b/fileproc/resource_monitor_metrics.go @@ -6,6 +6,8 @@ import ( "time" "github.com/sirupsen/logrus" + + "github.com/ivuorinen/gibidify/gibidiutils" ) // RecordFileProcessed records that a file has been successfully processed. @@ -55,7 +57,7 @@ func (rm *ResourceMonitor) GetMetrics() ResourceMetrics { ProcessingDuration: duration, AverageFileSize: avgFileSize, ProcessingRate: processingRate, - MemoryUsageMB: int64(m.Alloc) / 1024 / 1024, + MemoryUsageMB: gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0) / 1024 / 1024, MaxMemoryUsageMB: int64(rm.hardMemoryLimitMB), ViolationsDetected: violations, DegradationActive: rm.degradationActive, @@ -67,8 +69,13 @@ func (rm *ResourceMonitor) GetMetrics() ResourceMetrics { // LogResourceInfo logs current resource limit configuration. func (rm *ResourceMonitor) LogResourceInfo() { if rm.enabled { - logrus.Infof("Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds", - rm.maxFiles, rm.maxTotalSize/1024/1024, int(rm.fileProcessingTimeout.Seconds()), int(rm.overallTimeout.Seconds())) + logrus.Infof( + "Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds", + rm.maxFiles, + rm.maxTotalSize/1024/1024, + int(rm.fileProcessingTimeout.Seconds()), + int(rm.overallTimeout.Seconds()), + ) logrus.Infof("Resource limits: maxConcurrentReads=%d, rateLimitFPS=%d, hardMemoryMB=%d", rm.maxConcurrentReads, rm.rateLimitFilesPerSec, rm.hardMemoryLimitMB) logrus.Infof("Resource features: gracefulDegradation=%v, monitoring=%v", @@ -76,4 +83,4 @@ func (rm *ResourceMonitor) LogResourceInfo() { } else { logrus.Info("Resource limits disabled") } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_metrics_test.go b/fileproc/resource_monitor_metrics_test.go index b804581..1b28786 100644 --- a/fileproc/resource_monitor_metrics_test.go +++ b/fileproc/resource_monitor_metrics_test.go @@ -46,4 +46,4 @@ func TestResourceMonitor_Metrics(t *testing.T) { if !metrics.LastUpdated.After(time.Now().Add(-time.Second)) { t.Error("Expected recent LastUpdated timestamp") } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_rate_limiting.go b/fileproc/resource_monitor_rate_limiting.go index 640eab4..c475777 100644 --- a/fileproc/resource_monitor_rate_limiting.go +++ b/fileproc/resource_monitor_rate_limiting.go @@ -33,4 +33,4 @@ func (rm *ResourceMonitor) rateLimiterRefill() { // Channel is full, skip } } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_rate_limiting_test.go b/fileproc/resource_monitor_rate_limiting_test.go index d5c791d..4c8e15d 100644 --- a/fileproc/resource_monitor_rate_limiting_test.go +++ b/fileproc/resource_monitor_rate_limiting_test.go @@ -37,4 +37,4 @@ func TestResourceMonitor_RateLimiting(t *testing.T) { if duration < 200*time.Millisecond { t.Logf("Rate limiting may not be working as expected, took only %v", duration) } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_state.go b/fileproc/resource_monitor_state.go index e1abef7..1fe544e 100644 --- a/fileproc/resource_monitor_state.go +++ b/fileproc/resource_monitor_state.go @@ -19,4 +19,4 @@ func (rm *ResourceMonitor) Close() { if rm.rateLimiter != nil { rm.rateLimiter.Stop() } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_types.go b/fileproc/resource_monitor_types.go index 90461b9..5b12758 100644 --- a/fileproc/resource_monitor_types.go +++ b/fileproc/resource_monitor_types.go @@ -100,9 +100,9 @@ func NewResourceMonitor() *ResourceMonitor { } rateLimitFull: - // Start rate limiter refill goroutine + // Start rate limiter refill goroutine go rm.rateLimiterRefill() } return rm -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_types_test.go b/fileproc/resource_monitor_types_test.go index 8686aca..7d91c44 100644 --- a/fileproc/resource_monitor_types_test.go +++ b/fileproc/resource_monitor_types_test.go @@ -34,7 +34,7 @@ func TestResourceMonitor_NewResourceMonitor(t *testing.T) { } if rm.fileProcessingTimeout != time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second { - t.Errorf("Expected fileProcessingTimeout to be %v, got %v", + t.Errorf("Expected fileProcessingTimeout to be %v, got %v", time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second, rm.fileProcessingTimeout) } @@ -71,4 +71,4 @@ func TestResourceMonitor_DisabledResourceLimits(t *testing.T) { if err != nil { t.Errorf("Expected no error when rate limiting disabled, got %v", err) } -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_validation.go b/fileproc/resource_monitor_validation.go index f24dad8..7170862 100644 --- a/fileproc/resource_monitor_validation.go +++ b/fileproc/resource_monitor_validation.go @@ -7,7 +7,7 @@ import ( "github.com/sirupsen/logrus" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // ValidateFileProcessing checks if a file can be processed based on resource limits. @@ -21,9 +21,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6 // Check if emergency stop is active if rm.emergencyStopRequested { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitMemory, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitMemory, "processing stopped due to emergency memory condition", filePath, map[string]interface{}{ @@ -35,9 +35,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6 // Check file count limit currentFiles := atomic.LoadInt64(&rm.filesProcessed) if int(currentFiles) >= rm.maxFiles { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitFiles, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitFiles, "maximum file count limit exceeded", filePath, map[string]interface{}{ @@ -50,9 +50,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6 // Check total size limit currentTotalSize := atomic.LoadInt64(&rm.totalSizeProcessed) if currentTotalSize+fileSize > rm.maxTotalSize { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitTotalSize, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitTotalSize, "maximum total size limit would be exceeded", filePath, map[string]interface{}{ @@ -65,9 +65,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6 // Check overall timeout if time.Since(rm.startTime) > rm.overallTimeout { - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitTimeout, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitTimeout, "overall processing timeout exceeded", filePath, map[string]interface{}{ @@ -88,7 +88,7 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error { var m runtime.MemStats runtime.ReadMemStats(&m) - currentMemory := int64(m.Alloc) + currentMemory := gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0) if currentMemory > rm.hardMemoryLimitBytes { rm.mu.Lock() @@ -108,14 +108,14 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error { // Check again after GC runtime.ReadMemStats(&m) - currentMemory = int64(m.Alloc) + currentMemory = gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0) if currentMemory > rm.hardMemoryLimitBytes { // Still over limit, activate emergency stop rm.emergencyStopRequested = true - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitMemory, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitMemory, "hard memory limit exceeded, emergency stop activated", "", map[string]interface{}{ @@ -124,16 +124,15 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error { "emergency_stop": true, }, ) - } else { - // Memory freed by GC, continue with degradation - rm.degradationActive = true - logrus.Info("Memory freed by garbage collection, continuing with degradation mode") } + // Memory freed by GC, continue with degradation + rm.degradationActive = true + logrus.Info("Memory freed by garbage collection, continuing with degradation mode") } else { // No graceful degradation, hard stop - return utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeResourceLimitMemory, + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeResourceLimitMemory, "hard memory limit exceeded", "", map[string]interface{}{ @@ -145,4 +144,4 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error { } return nil -} \ No newline at end of file +} diff --git a/fileproc/resource_monitor_validation_test.go b/fileproc/resource_monitor_validation_test.go index d45002c..26c675c 100644 --- a/fileproc/resource_monitor_validation_test.go +++ b/fileproc/resource_monitor_validation_test.go @@ -1,12 +1,13 @@ package fileproc import ( + "errors" "testing" "github.com/spf13/viper" + "github.com/ivuorinen/gibidify/gibidiutils" "github.com/ivuorinen/gibidify/testutil" - "github.com/ivuorinen/gibidify/utils" ) func TestResourceMonitor_FileCountLimit(t *testing.T) { @@ -40,11 +41,12 @@ func TestResourceMonitor_FileCountLimit(t *testing.T) { } // Verify it's the correct error type - structErr, ok := err.(*utils.StructuredError) + var structErr *gibidiutils.StructuredError + ok := errors.As(err, &structErr) if !ok { t.Errorf("Expected StructuredError, got %T", err) - } else if structErr.Code != utils.CodeResourceLimitFiles { - t.Errorf("Expected error code %s, got %s", utils.CodeResourceLimitFiles, structErr.Code) + } else if structErr.Code != gibidiutils.CodeResourceLimitFiles { + t.Errorf("Expected error code %s, got %s", gibidiutils.CodeResourceLimitFiles, structErr.Code) } } @@ -79,10 +81,11 @@ func TestResourceMonitor_TotalSizeLimit(t *testing.T) { } // Verify it's the correct error type - structErr, ok := err.(*utils.StructuredError) + var structErr *gibidiutils.StructuredError + ok := errors.As(err, &structErr) if !ok { t.Errorf("Expected StructuredError, got %T", err) - } else if structErr.Code != utils.CodeResourceLimitTotalSize { - t.Errorf("Expected error code %s, got %s", utils.CodeResourceLimitTotalSize, structErr.Code) + } else if structErr.Code != gibidiutils.CodeResourceLimitTotalSize { + t.Errorf("Expected error code %s, got %s", gibidiutils.CodeResourceLimitTotalSize, structErr.Code) } -} \ No newline at end of file +} diff --git a/fileproc/test_constants.go b/fileproc/test_constants.go new file mode 100644 index 0000000..9ecdc5e --- /dev/null +++ b/fileproc/test_constants.go @@ -0,0 +1,12 @@ +package fileproc + +// Test constants to avoid duplication in test files. +// These constants are used across multiple test files in the fileproc package. +const ( + // Backpressure configuration keys + testBackpressureEnabled = "backpressure.enabled" + testBackpressureMaxMemory = "backpressure.maxMemoryUsage" + testBackpressureMemoryCheck = "backpressure.memoryCheckInterval" + testBackpressureMaxFiles = "backpressure.maxPendingFiles" + testBackpressureMaxWrites = "backpressure.maxPendingWrites" +) diff --git a/fileproc/walker.go b/fileproc/walker.go index 58f9e64..95ea0b8 100644 --- a/fileproc/walker.go +++ b/fileproc/walker.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // Walker defines an interface for scanning directories. @@ -30,9 +30,12 @@ func NewProdWalker() *ProdWalker { // Walk scans the given root directory recursively and returns a slice of file paths // that are not ignored based on .gitignore/.ignore files, the configuration, or the default binary/image filter. func (w *ProdWalker) Walk(root string) ([]string, error) { - absRoot, err := utils.GetAbsolutePath(root) + absRoot, err := gibidiutils.GetAbsolutePath(root) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSPathResolution, "failed to resolve root path").WithFilePath(root) + return nil, gibidiutils.WrapError( + err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSPathResolution, + "failed to resolve root path", + ).WithFilePath(root) } return w.walkDir(absRoot, []ignoreRule{}) } @@ -47,7 +50,10 @@ func (w *ProdWalker) walkDir(currentDir string, parentRules []ignoreRule) ([]str entries, err := os.ReadDir(currentDir) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to read directory").WithFilePath(currentDir) + return nil, gibidiutils.WrapError( + err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSAccess, + "failed to read directory", + ).WithFilePath(currentDir) } rules := loadIgnoreRules(currentDir, parentRules) @@ -63,7 +69,10 @@ func (w *ProdWalker) walkDir(currentDir string, parentRules []ignoreRule) ([]str if entry.IsDir() { subFiles, err := w.walkDir(fullPath, rules) if err != nil { - return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingTraversal, "failed to traverse subdirectory").WithFilePath(fullPath) + return nil, gibidiutils.WrapError( + err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingTraversal, + "failed to traverse subdirectory", + ).WithFilePath(fullPath) } results = append(results, subFiles...) } else { diff --git a/fileproc/walker_test.go b/fileproc/walker_test.go index dfee038..bf8ef81 100644 --- a/fileproc/walker_test.go +++ b/fileproc/walker_test.go @@ -61,6 +61,8 @@ func TestProdWalkerBinaryCheck(t *testing.T) { // Reset FileTypeRegistry to ensure clean state fileproc.ResetRegistryForTesting() + // Ensure cleanup runs even if test fails + t.Cleanup(fileproc.ResetRegistryForTesting) // Run walker w := fileproc.NewProdWalker() diff --git a/fileproc/writer.go b/fileproc/writer.go index 303aaf2..66192b2 100644 --- a/fileproc/writer.go +++ b/fileproc/writer.go @@ -5,30 +5,100 @@ import ( "fmt" "os" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) -// StartWriter writes the output in the specified format with memory optimization. -func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, format, prefix, suffix string) { - switch format { - case "markdown": - startMarkdownWriter(outFile, writeCh, done, prefix, suffix) - case "json": - startJSONWriter(outFile, writeCh, done, prefix, suffix) - case "yaml": - startYAMLWriter(outFile, writeCh, done, prefix, suffix) +// WriterConfig holds configuration for the writer. +type WriterConfig struct { + Format string + Prefix string + Suffix string +} + +// Validate checks if the WriterConfig is valid. +func (c WriterConfig) Validate() error { + if c.Format == "" { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + "format cannot be empty", + "", + nil, + ) + } + + switch c.Format { + case "markdown", "json", "yaml": + return nil default: - context := map[string]interface{}{ - "format": format, + context := map[string]any{ + "format": c.Format, } - err := utils.NewStructuredError( - utils.ErrorTypeValidation, - utils.CodeValidationFormat, - fmt.Sprintf("unsupported format: %s", format), + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + fmt.Sprintf("unsupported format: %s", c.Format), "", context, ) - utils.LogError("Failed to encode output", err) + } +} + +// StartWriter writes the output in the specified format with memory optimization. +func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, config WriterConfig) { + // Validate config + if err := config.Validate(); err != nil { + gibidiutils.LogError("Invalid writer configuration", err) + close(done) + return + } + + // Validate outFile is not nil + if outFile == nil { + err := gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileWrite, + "output file is nil", + "", + nil, + ) + gibidiutils.LogError("Failed to write output", err) + close(done) + return + } + + // Validate outFile is accessible + if _, err := outFile.Stat(); err != nil { + structErr := gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOFileWrite, + "failed to stat output file", + ) + gibidiutils.LogError("Failed to validate output file", structErr) + close(done) + return + } + + switch config.Format { + case "markdown": + startMarkdownWriter(outFile, writeCh, done, config.Prefix, config.Suffix) + case "json": + startJSONWriter(outFile, writeCh, done, config.Prefix, config.Suffix) + case "yaml": + startYAMLWriter(outFile, writeCh, done, config.Prefix, config.Suffix) + default: + context := map[string]interface{}{ + "format": config.Format, + } + err := gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationFormat, + fmt.Sprintf("unsupported format: %s", config.Format), + "", + context, + ) + gibidiutils.LogError("Failed to encode output", err) close(done) } } diff --git a/fileproc/writer_test.go b/fileproc/writer_test.go index 0320e23..88313ce 100644 --- a/fileproc/writer_test.go +++ b/fileproc/writer_test.go @@ -68,7 +68,11 @@ func runWriterTest(t *testing.T, format string) []byte { wg.Add(1) go func() { defer wg.Done() - fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX") + fileproc.StartWriter(outFile, writeCh, doneCh, fileproc.WriterConfig{ + Format: format, + Prefix: "PREFIX", + Suffix: "SUFFIX", + }) }() // Wait until writer signals completion diff --git a/fileproc/yaml_writer.go b/fileproc/yaml_writer.go index 3ea60f2..0a4dc75 100644 --- a/fileproc/yaml_writer.go +++ b/fileproc/yaml_writer.go @@ -5,9 +5,10 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" - "github.com/ivuorinen/gibidify/utils" + "github.com/ivuorinen/gibidify/gibidiutils" ) // YAMLWriter handles YAML format output with streaming support. @@ -20,11 +21,151 @@ func NewYAMLWriter(outFile *os.File) *YAMLWriter { return &YAMLWriter{outFile: outFile} } +const ( + maxPathLength = 4096 // Maximum total path length + maxFilenameLength = 255 // Maximum individual filename component length +) + +// validatePathComponents validates individual path components for security issues. +func validatePathComponents(trimmed, cleaned string, components []string) error { + for i, component := range components { + // Reject path components that are exactly ".." (path traversal) + if component == ".." { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path traversal not allowed", + trimmed, + map[string]any{ + "path": trimmed, + "cleaned": cleaned, + "invalid_component": component, + "component_index": i, + }, + ) + } + + // Reject empty components (e.g., from "foo//bar") + if component == "" && i > 0 && i < len(components)-1 { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path contains empty component", + trimmed, + map[string]any{ + "path": trimmed, + "cleaned": cleaned, + "component_index": i, + }, + ) + } + + // Enforce maximum filename length for each component + if len(component) > maxFilenameLength { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path component exceeds maximum length", + trimmed, + map[string]any{ + "component": component, + "component_length": len(component), + "max_length": maxFilenameLength, + "component_index": i, + }, + ) + } + } + return nil +} + +// validatePath validates and sanitizes a file path for safe output. +// It rejects absolute paths, path traversal attempts, empty paths, and overly long paths. +func validatePath(path string) error { + // Reject empty paths + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationRequired, + "file path cannot be empty", + "", + nil, + ) + } + + // Enforce maximum path length to prevent resource abuse + if len(trimmed) > maxPathLength { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path exceeds maximum length", + trimmed, + map[string]any{ + "path_length": len(trimmed), + "max_length": maxPathLength, + }, + ) + } + + // Reject absolute paths + if filepath.IsAbs(trimmed) { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "absolute paths are not allowed", + trimmed, + map[string]any{"path": trimmed}, + ) + } + + // Validate original trimmed path components before cleaning + origComponents := strings.Split(filepath.ToSlash(trimmed), "/") + for _, comp := range origComponents { + if comp == "" || comp == "." || comp == ".." { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "invalid or traversal path component in original path", + trimmed, + map[string]any{"path": trimmed, "component": comp}, + ) + } + } + + // Clean the path to normalize it + cleaned := filepath.Clean(trimmed) + + // After cleaning, ensure it's still relative and doesn't start with / + if filepath.IsAbs(cleaned) || strings.HasPrefix(cleaned, "/") { + return gibidiutils.NewStructuredError( + gibidiutils.ErrorTypeValidation, + gibidiutils.CodeValidationPath, + "path must be relative", + trimmed, + map[string]any{"path": trimmed, "cleaned": cleaned}, + ) + } + + // Split into components and validate each one + // Use ToSlash to normalize for cross-platform validation + components := strings.Split(filepath.ToSlash(cleaned), "/") + return validatePathComponents(trimmed, cleaned, components) +} + // Start writes the YAML header. func (w *YAMLWriter) Start(prefix, suffix string) error { // Write YAML header - if _, err := fmt.Fprintf(w.outFile, "prefix: %s\nsuffix: %s\nfiles:\n", yamlQuoteString(prefix), yamlQuoteString(suffix)); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML header") + if _, err := fmt.Fprintf( + w.outFile, "prefix: %s\nsuffix: %s\nfiles:\n", + gibidiutils.EscapeForYAML(prefix), gibidiutils.EscapeForYAML(suffix), + ); err != nil { + return gibidiutils.WrapError( + err, + gibidiutils.ErrorTypeIO, + gibidiutils.CodeIOWrite, + "failed to write YAML header", + ) } return nil } @@ -44,13 +185,32 @@ func (w *YAMLWriter) Close() error { // writeStreaming writes a large file as YAML in streaming chunks. func (w *YAMLWriter) writeStreaming(req WriteRequest) error { - defer w.closeReader(req.Reader, req.Path) + // Validate path before using it + if err := validatePath(req.Path); err != nil { + return err + } + + // Check for nil reader + if req.Reader == nil { + return gibidiutils.WrapError( + nil, gibidiutils.ErrorTypeValidation, gibidiutils.CodeValidationRequired, + "nil reader in write request", + ).WithFilePath(req.Path) + } + + defer gibidiutils.SafeCloseReader(req.Reader, req.Path) language := detectLanguage(req.Path) // Write YAML file entry start - if _, err := fmt.Fprintf(w.outFile, " - path: %s\n language: %s\n content: |\n", yamlQuoteString(req.Path), language); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML file start").WithFilePath(req.Path) + if _, err := fmt.Fprintf( + w.outFile, " - path: %s\n language: %s\n content: |\n", + gibidiutils.EscapeForYAML(req.Path), language, + ); err != nil { + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write YAML file start", + ).WithFilePath(req.Path) } // Stream content with YAML indentation @@ -59,6 +219,11 @@ func (w *YAMLWriter) writeStreaming(req WriteRequest) error { // writeInline writes a small file directly as YAML. func (w *YAMLWriter) writeInline(req WriteRequest) error { + // Validate path before using it + if err := validatePath(req.Path); err != nil { + return err + } + language := detectLanguage(req.Path) fileData := FileData{ Path: req.Path, @@ -67,15 +232,24 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error { } // Write YAML entry - if _, err := fmt.Fprintf(w.outFile, " - path: %s\n language: %s\n content: |\n", yamlQuoteString(fileData.Path), fileData.Language); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML entry start").WithFilePath(req.Path) + if _, err := fmt.Fprintf( + w.outFile, " - path: %s\n language: %s\n content: |\n", + gibidiutils.EscapeForYAML(fileData.Path), fileData.Language, + ); err != nil { + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write YAML entry start", + ).WithFilePath(req.Path) } // Write indented content lines := strings.Split(fileData.Content, "\n") for _, line := range lines { if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML content line").WithFilePath(req.Path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write YAML content line", + ).WithFilePath(req.Path) } } @@ -85,43 +259,29 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error { // streamYAMLContent streams content with YAML indentation. func (w *YAMLWriter) streamYAMLContent(reader io.Reader, path string) error { scanner := bufio.NewScanner(reader) + // Increase buffer size to handle long lines (up to 10MB per line) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 10*1024*1024) + for scanner.Scan() { line := scanner.Text() if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML line").WithFilePath(path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, + "failed to write YAML line", + ).WithFilePath(path) } } if err := scanner.Err(); err != nil { - return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIORead, "failed to scan YAML content").WithFilePath(path) + return gibidiutils.WrapError( + err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOFileRead, + "failed to scan YAML content", + ).WithFilePath(path) } return nil } -// closeReader safely closes a reader if it implements io.Closer. -func (w *YAMLWriter) closeReader(reader io.Reader, path string) { - if closer, ok := reader.(io.Closer); ok { - if err := closer.Close(); err != nil { - utils.LogError( - "Failed to close file reader", - utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOClose, "failed to close file reader").WithFilePath(path), - ) - } - } -} - -// yamlQuoteString quotes a string for YAML output if needed. -func yamlQuoteString(s string) string { - if s == "" { - return `""` - } - // Simple YAML quoting - use double quotes if string contains special characters - if strings.ContainsAny(s, "\n\r\t:\"'\\") { - return fmt.Sprintf(`"%s"`, strings.ReplaceAll(s, `"`, `\"`)) - } - return s -} - // startYAMLWriter handles YAML format output with streaming support. func startYAMLWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) { defer close(done) @@ -130,19 +290,19 @@ func startYAMLWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- // Start writing if err := writer.Start(prefix, suffix); err != nil { - utils.LogError("Failed to write YAML header", err) + gibidiutils.LogError("Failed to write YAML header", err) return } // Process files for req := range writeCh { if err := writer.WriteFile(req); err != nil { - utils.LogError("Failed to write YAML file", err) + gibidiutils.LogError("Failed to write YAML file", err) } } // Close writer if err := writer.Close(); err != nil { - utils.LogError("Failed to write YAML end", err) + gibidiutils.LogError("Failed to write YAML end", err) } } diff --git a/utils/errors.go b/gibidiutils/errors.go similarity index 81% rename from utils/errors.go rename to gibidiutils/errors.go index 9b0c456..c41ebcd 100644 --- a/utils/errors.go +++ b/gibidiutils/errors.go @@ -1,8 +1,11 @@ -// Package utils provides common utility functions. -package utils +// Package gibidiutils provides common utility functions for gibidify. +package gibidiutils import ( + "errors" "fmt" + "sort" + "strings" "github.com/sirupsen/logrus" ) @@ -47,6 +50,11 @@ func (e ErrorType) String() string { } } +// Error formatting templates. +const ( + errorFormatWithCause = "%s: %v" +) + // StructuredError represents a structured error with type, code, and context. type StructuredError struct { Type ErrorType @@ -60,10 +68,25 @@ type StructuredError struct { // Error implements the error interface. func (e *StructuredError) Error() string { - if e.Cause != nil { - return fmt.Sprintf("%s [%s]: %s: %v", e.Type, e.Code, e.Message, e.Cause) + base := fmt.Sprintf("%s [%s]: %s", e.Type, e.Code, e.Message) + if len(e.Context) > 0 { + // Sort keys for deterministic output + keys := make([]string, 0, len(e.Context)) + for k := range e.Context { + keys = append(keys, k) + } + sort.Strings(keys) + + ctxPairs := make([]string, 0, len(e.Context)) + for _, k := range keys { + ctxPairs = append(ctxPairs, fmt.Sprintf("%s=%v", k, e.Context[k])) + } + base = fmt.Sprintf("%s | context: %s", base, strings.Join(ctxPairs, ", ")) } - return fmt.Sprintf("%s [%s]: %s", e.Type, e.Code, e.Message) + if e.Cause != nil { + return fmt.Sprintf(errorFormatWithCause, base, e.Cause) + } + return base } // Unwrap returns the underlying cause error. @@ -93,7 +116,11 @@ func (e *StructuredError) WithLine(line int) *StructuredError { } // NewStructuredError creates a new structured error. -func NewStructuredError(errorType ErrorType, code, message, filePath string, context map[string]interface{}) *StructuredError { +func NewStructuredError( + errorType ErrorType, + code, message, filePath string, + context map[string]any, +) *StructuredError { return &StructuredError{ Type: errorType, Code: code, @@ -135,34 +162,40 @@ func WrapErrorf(err error, errorType ErrorType, code, format string, args ...any // Common error codes for each type const ( // CLI Error Codes + CodeCLIMissingSource = "MISSING_SOURCE" CodeCLIInvalidArgs = "INVALID_ARGS" // FileSystem Error Codes + CodeFSPathResolution = "PATH_RESOLUTION" CodeFSPermission = "PERMISSION_DENIED" CodeFSNotFound = "NOT_FOUND" CodeFSAccess = "ACCESS_DENIED" // Processing Error Codes + CodeProcessingFileRead = "FILE_READ" CodeProcessingCollection = "COLLECTION" CodeProcessingTraversal = "TRAVERSAL" CodeProcessingEncode = "ENCODE" // Configuration Error Codes + CodeConfigValidation = "VALIDATION" CodeConfigMissing = "MISSING" // IO Error Codes + CodeIOFileCreate = "FILE_CREATE" CodeIOFileWrite = "FILE_WRITE" CodeIOEncoding = "ENCODING" CodeIOWrite = "WRITE" - CodeIORead = "READ" + CodeIOFileRead = "FILE_READ" CodeIOClose = "CLOSE" // Validation Error Codes + CodeValidationFormat = "FORMAT" CodeValidationFileType = "FILE_TYPE" CodeValidationSize = "SIZE_LIMIT" @@ -170,6 +203,7 @@ const ( CodeValidationPath = "PATH_TRAVERSAL" // Resource Limit Error Codes + CodeResourceLimitFiles = "FILE_COUNT_LIMIT" CodeResourceLimitTotalSize = "TOTAL_SIZE_LIMIT" CodeResourceLimitTimeout = "TIMEOUT" @@ -180,9 +214,16 @@ const ( // Predefined error constructors for common error scenarios -// NewCLIMissingSourceError creates a CLI error for missing source argument. -func NewCLIMissingSourceError() *StructuredError { - return NewStructuredError(ErrorTypeCLI, CodeCLIMissingSource, "usage: gibidify -source [--destination ] [--format=json|yaml|markdown]", "", nil) +// NewMissingSourceError creates a CLI error for missing source argument. +func NewMissingSourceError() *StructuredError { + return NewStructuredError( + ErrorTypeCLI, + CodeCLIMissingSource, + "usage: gibidify -source "+ + "[--destination ] [--format=json|yaml|markdown]", + "", + nil, + ) } // NewFileSystemError creates a file system error. @@ -217,16 +258,18 @@ func LogError(operation string, err error, args ...any) { } // Check if it's a structured error and log with additional context - if structErr, ok := err.(*StructuredError); ok { + var structErr *StructuredError + if errors.As(err, &structErr) { logrus.WithFields(logrus.Fields{ "error_type": structErr.Type.String(), "error_code": structErr.Code, "context": structErr.Context, "file_path": structErr.FilePath, "line": structErr.Line, - }).Errorf("%s: %v", msg, err) + }).Errorf(errorFormatWithCause, msg, err) } else { - logrus.Errorf("%s: %v", msg, err) + // Log regular errors without structured fields + logrus.Errorf(errorFormatWithCause, msg, err) } } } diff --git a/gibidiutils/errors_additional_test.go b/gibidiutils/errors_additional_test.go new file mode 100644 index 0000000..8d42f6b --- /dev/null +++ b/gibidiutils/errors_additional_test.go @@ -0,0 +1,367 @@ +package gibidiutils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorTypeString(t *testing.T) { + tests := []struct { + name string + errType ErrorType + expected string + }{ + { + name: "CLI error type", + errType: ErrorTypeCLI, + expected: "CLI", + }, + { + name: "FileSystem error type", + errType: ErrorTypeFileSystem, + expected: "FileSystem", + }, + { + name: "Processing error type", + errType: ErrorTypeProcessing, + expected: "Processing", + }, + { + name: "Configuration error type", + errType: ErrorTypeConfiguration, + expected: "Configuration", + }, + { + name: "IO error type", + errType: ErrorTypeIO, + expected: "IO", + }, + { + name: "Validation error type", + errType: ErrorTypeValidation, + expected: "Validation", + }, + { + name: "Unknown error type", + errType: ErrorTypeUnknown, + expected: "Unknown", + }, + { + name: "Invalid error type", + errType: ErrorType(999), + expected: "Unknown", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.errType.String() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestStructuredErrorMethods(t *testing.T) { + t.Run("Error method", func(t *testing.T) { + err := &StructuredError{ + Type: ErrorTypeValidation, + Code: CodeValidationRequired, + Message: "field is required", + } + expected := "Validation [REQUIRED]: field is required" + assert.Equal(t, expected, err.Error()) + }) + + t.Run("Error method with context", func(t *testing.T) { + err := &StructuredError{ + Type: ErrorTypeFileSystem, + Code: CodeFSNotFound, + Message: testErrFileNotFound, + Context: map[string]interface{}{ + "path": "/test/file.txt", + }, + } + errStr := err.Error() + assert.Contains(t, errStr, "FileSystem") + assert.Contains(t, errStr, "NOT_FOUND") + assert.Contains(t, errStr, testErrFileNotFound) + assert.Contains(t, errStr, "/test/file.txt") + assert.Contains(t, errStr, "path") + }) + + t.Run("Unwrap method", func(t *testing.T) { + innerErr := errors.New("inner error") + err := &StructuredError{ + Type: ErrorTypeIO, + Code: CodeIOFileWrite, + Message: testErrWriteFailed, + Cause: innerErr, + } + assert.Equal(t, innerErr, err.Unwrap()) + }) + + t.Run("Unwrap with nil cause", func(t *testing.T) { + err := &StructuredError{ + Type: ErrorTypeIO, + Code: CodeIOFileWrite, + Message: testErrWriteFailed, + } + assert.Nil(t, err.Unwrap()) + }) +} + +func TestWithContextMethods(t *testing.T) { + t.Run("WithContext", func(t *testing.T) { + err := &StructuredError{ + Type: ErrorTypeValidation, + Code: CodeValidationFormat, + Message: testErrInvalidFormat, + } + + err = err.WithContext("format", "xml") + err = err.WithContext("expected", "json") + + assert.NotNil(t, err.Context) + assert.Equal(t, "xml", err.Context["format"]) + assert.Equal(t, "json", err.Context["expected"]) + }) + + t.Run("WithFilePath", func(t *testing.T) { + err := &StructuredError{ + Type: ErrorTypeFileSystem, + Code: CodeFSPermission, + Message: "permission denied", + } + + err = err.WithFilePath("/etc/passwd") + + assert.Equal(t, "/etc/passwd", err.FilePath) + }) + + t.Run("WithLine", func(t *testing.T) { + err := &StructuredError{ + Type: ErrorTypeProcessing, + Code: CodeProcessingFileRead, + Message: "read error", + } + + err = err.WithLine(42) + + assert.Equal(t, 42, err.Line) + }) +} + +func TestNewStructuredError(t *testing.T) { + tests := []struct { + name string + errType ErrorType + code string + message string + filePath string + context map[string]interface{} + }{ + { + name: "basic error", + errType: ErrorTypeValidation, + code: CodeValidationRequired, + message: "field is required", + filePath: "", + context: nil, + }, + { + name: "error with file path", + errType: ErrorTypeFileSystem, + code: CodeFSNotFound, + message: testErrFileNotFound, + filePath: "/test/missing.txt", + context: nil, + }, + { + name: "error with context", + errType: ErrorTypeIO, + code: CodeIOFileWrite, + message: testErrWriteFailed, + context: map[string]interface{}{ + "size": 1024, + "error": "disk full", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := NewStructuredError(tt.errType, tt.code, tt.message, tt.filePath, tt.context) + + assert.NotNil(t, err) + assert.Equal(t, tt.errType, err.Type) + assert.Equal(t, tt.code, err.Code) + assert.Equal(t, tt.message, err.Message) + assert.Equal(t, tt.filePath, err.FilePath) + assert.Equal(t, tt.context, err.Context) + }) + } +} + +func TestNewStructuredErrorf(t *testing.T) { + err := NewStructuredErrorf( + ErrorTypeValidation, + CodeValidationSize, + "file size %d exceeds limit %d", + 2048, 1024, + ) + + assert.NotNil(t, err) + assert.Equal(t, ErrorTypeValidation, err.Type) + assert.Equal(t, CodeValidationSize, err.Code) + assert.Equal(t, "file size 2048 exceeds limit 1024", err.Message) +} + +func TestWrapError(t *testing.T) { + innerErr := errors.New("original error") + wrappedErr := WrapError( + innerErr, + ErrorTypeProcessing, + CodeProcessingFileRead, + "failed to process file", + ) + + assert.NotNil(t, wrappedErr) + assert.Equal(t, ErrorTypeProcessing, wrappedErr.Type) + assert.Equal(t, CodeProcessingFileRead, wrappedErr.Code) + assert.Equal(t, "failed to process file", wrappedErr.Message) + assert.Equal(t, innerErr, wrappedErr.Cause) +} + +func TestWrapErrorf(t *testing.T) { + innerErr := errors.New("original error") + wrappedErr := WrapErrorf( + innerErr, + ErrorTypeIO, + CodeIOFileCreate, + "failed to create %s in %s", + "output.txt", "/tmp", + ) + + assert.NotNil(t, wrappedErr) + assert.Equal(t, ErrorTypeIO, wrappedErr.Type) + assert.Equal(t, CodeIOFileCreate, wrappedErr.Code) + assert.Equal(t, "failed to create output.txt in /tmp", wrappedErr.Message) + assert.Equal(t, innerErr, wrappedErr.Cause) +} + +func TestSpecificErrorConstructors(t *testing.T) { + t.Run("NewMissingSourceError", func(t *testing.T) { + err := NewMissingSourceError() + assert.NotNil(t, err) + assert.Equal(t, ErrorTypeCLI, err.Type) + assert.Equal(t, CodeCLIMissingSource, err.Code) + assert.Contains(t, err.Message, "source") + }) + + t.Run("NewFileSystemError", func(t *testing.T) { + err := NewFileSystemError(CodeFSPermission, "access denied") + assert.NotNil(t, err) + assert.Equal(t, ErrorTypeFileSystem, err.Type) + assert.Equal(t, CodeFSPermission, err.Code) + assert.Equal(t, "access denied", err.Message) + }) + + t.Run("NewProcessingError", func(t *testing.T) { + err := NewProcessingError(CodeProcessingCollection, "collection failed") + assert.NotNil(t, err) + assert.Equal(t, ErrorTypeProcessing, err.Type) + assert.Equal(t, CodeProcessingCollection, err.Code) + assert.Equal(t, "collection failed", err.Message) + }) + + t.Run("NewIOError", func(t *testing.T) { + err := NewIOError(CodeIOFileWrite, testErrWriteFailed) + assert.NotNil(t, err) + assert.Equal(t, ErrorTypeIO, err.Type) + assert.Equal(t, CodeIOFileWrite, err.Code) + assert.Equal(t, testErrWriteFailed, err.Message) + }) + + t.Run("NewValidationError", func(t *testing.T) { + err := NewValidationError(CodeValidationFormat, testErrInvalidFormat) + assert.NotNil(t, err) + assert.Equal(t, ErrorTypeValidation, err.Type) + assert.Equal(t, CodeValidationFormat, err.Code) + assert.Equal(t, testErrInvalidFormat, err.Message) + }) +} + +// TestLogErrorf is already covered in errors_test.go + +func TestStructuredErrorChaining(t *testing.T) { + // Test method chaining + err := NewStructuredError( + ErrorTypeFileSystem, + CodeFSNotFound, + testErrFileNotFound, + "", + nil, + ).WithFilePath("/test.txt").WithLine(10).WithContext("operation", "read") + + assert.Equal(t, "/test.txt", err.FilePath) + assert.Equal(t, 10, err.Line) + assert.Equal(t, "read", err.Context["operation"]) +} + +func TestErrorCodes(t *testing.T) { + // Test that all error codes are defined + codes := []string{ + CodeCLIMissingSource, + CodeCLIInvalidArgs, + CodeFSPathResolution, + CodeFSPermission, + CodeFSNotFound, + CodeFSAccess, + CodeProcessingFileRead, + CodeProcessingCollection, + CodeProcessingTraversal, + CodeProcessingEncode, + CodeConfigValidation, + CodeConfigMissing, + CodeIOFileCreate, + CodeIOFileWrite, + CodeIOEncoding, + CodeIOWrite, + CodeIOFileRead, + CodeIOClose, + CodeValidationRequired, + CodeValidationFormat, + CodeValidationSize, + CodeValidationPath, + CodeResourceLimitFiles, + CodeResourceLimitTotalSize, + CodeResourceLimitMemory, + CodeResourceLimitTimeout, + } + + // All codes should be non-empty strings + for _, code := range codes { + assert.NotEmpty(t, code, "Error code should not be empty") + assert.NotEqual(t, "", code, "Error code should be defined") + } +} + +func TestErrorUnwrapChain(t *testing.T) { + // Test unwrapping through multiple levels + innermost := errors.New("innermost error") + middle := WrapError(innermost, ErrorTypeIO, CodeIOFileRead, "read failed") + outer := WrapError(middle, ErrorTypeProcessing, CodeProcessingFileRead, "processing failed") + + // Test unwrapping + assert.Equal(t, middle, outer.Unwrap()) + assert.Equal(t, innermost, middle.Unwrap()) + + // innermost is a plain error, doesn't have Unwrap() method + // No need to test it + + // Test error chain messages + assert.Contains(t, outer.Error(), "Processing") + assert.Contains(t, middle.Error(), "IO") +} diff --git a/utils/errors_test.go b/gibidiutils/errors_test.go similarity index 96% rename from utils/errors_test.go rename to gibidiutils/errors_test.go index 1831240..c401f2f 100644 --- a/utils/errors_test.go +++ b/gibidiutils/errors_test.go @@ -1,4 +1,5 @@ -package utils +// Package gibidiutils provides common utility functions for gibidify. +package gibidiutils import ( "bytes" @@ -175,7 +176,7 @@ func TestLogErrorf(t *testing.T) { } } -func TestLogErrorConcurrency(t *testing.T) { +func TestLogErrorConcurrency(_ *testing.T) { // Test that LogError is safe for concurrent use done := make(chan bool) for i := 0; i < 10; i++ { @@ -191,7 +192,7 @@ func TestLogErrorConcurrency(t *testing.T) { } } -func TestLogErrorfConcurrency(t *testing.T) { +func TestLogErrorfConcurrency(_ *testing.T) { // Test that LogErrorf is safe for concurrent use done := make(chan bool) for i := 0; i < 10; i++ { diff --git a/gibidiutils/icons.go b/gibidiutils/icons.go new file mode 100644 index 0000000..75a3295 --- /dev/null +++ b/gibidiutils/icons.go @@ -0,0 +1,10 @@ +package gibidiutils + +// Unicode icons and symbols for CLI UI and test output. +const ( + IconSuccess = "✓" // U+2713 + IconError = "✗" // U+2717 + IconWarning = "⚠" // U+26A0 + IconBullet = "•" // U+2022 + IconInfo = "ℹ️" // U+2139 FE0F +) diff --git a/gibidiutils/paths.go b/gibidiutils/paths.go new file mode 100644 index 0000000..93d5568 --- /dev/null +++ b/gibidiutils/paths.go @@ -0,0 +1,311 @@ +// Package gibidiutils provides common utility functions for gibidify. +package gibidiutils + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// EscapeForMarkdown sanitizes a string for safe use in Markdown code-fence and header lines. +// It replaces backticks with backslash-escaped backticks and removes/collapses newlines. +func EscapeForMarkdown(s string) string { + // Escape backticks + safe := strings.ReplaceAll(s, "`", "\\`") + // Remove newlines (collapse to space) + safe = strings.ReplaceAll(safe, "\n", " ") + safe = strings.ReplaceAll(safe, "\r", " ") + return safe +} + +// GetAbsolutePath returns the absolute path for the given path. +// It wraps filepath.Abs with consistent error handling. +func GetAbsolutePath(path string) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to get absolute path for %s: %w", path, err) + } + return abs, nil +} + +// GetBaseName returns the base name for the given path, handling special cases. +func GetBaseName(absPath string) string { + baseName := filepath.Base(absPath) + if baseName == "." || baseName == "" { + return "output" + } + return baseName +} + +// checkPathTraversal checks for path traversal patterns and returns an error if found. +func checkPathTraversal(path, context string) error { + // Normalize separators without cleaning (to preserve ..) + normalized := filepath.ToSlash(path) + + // Split into components + components := strings.Split(normalized, "/") + + // Check each component for exact ".." match + for _, component := range components { + if component == ".." { + return NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + fmt.Sprintf("path traversal attempt detected in %s", context), + path, + map[string]interface{}{ + "original_path": path, + }, + ) + } + } + return nil +} + +// cleanAndResolveAbsPath cleans a path and resolves it to an absolute path. +func cleanAndResolveAbsPath(path, context string) (string, error) { + cleaned := filepath.Clean(path) + abs, err := filepath.Abs(cleaned) + if err != nil { + return "", NewStructuredError( + ErrorTypeFileSystem, + CodeFSPathResolution, + fmt.Sprintf("cannot resolve %s", context), + path, + map[string]interface{}{ + "error": err.Error(), + }, + ) + } + return abs, nil +} + +// evalSymlinksOrStructuredError wraps filepath.EvalSymlinks with structured error handling. +func evalSymlinksOrStructuredError(path, context, original string) (string, error) { + eval, err := filepath.EvalSymlinks(path) + if err != nil { + return "", NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + fmt.Sprintf("cannot resolve symlinks for %s", context), + original, + map[string]interface{}{ + "resolved_path": path, + "context": context, + "error": err.Error(), + }, + ) + } + return eval, nil +} + +// validateWorkingDirectoryBoundary checks if the given absolute path escapes the working directory. +func validateWorkingDirectoryBoundary(abs, path string) error { + cwd, err := os.Getwd() + if err != nil { + return NewStructuredError( + ErrorTypeFileSystem, + CodeFSPathResolution, + "cannot get current working directory", + path, + map[string]interface{}{ + "error": err.Error(), + }, + ) + } + + cwdAbs, err := filepath.Abs(cwd) + if err != nil { + return NewStructuredError( + ErrorTypeFileSystem, + CodeFSPathResolution, + "cannot resolve current working directory", + path, + map[string]interface{}{ + "error": err.Error(), + }, + ) + } + + absEval, err := evalSymlinksOrStructuredError(abs, "source path", path) + if err != nil { + return err + } + cwdEval, err := evalSymlinksOrStructuredError(cwdAbs, "working directory", path) + if err != nil { + return err + } + + rel, err := filepath.Rel(cwdEval, absEval) + if err != nil { + return NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + "cannot determine relative path", + path, + map[string]interface{}{ + "resolved_path": absEval, + "working_dir": cwdEval, + "error": err.Error(), + }, + ) + } + + if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + "source path attempts to access directories outside current working directory", + path, + map[string]interface{}{ + "resolved_path": absEval, + "working_dir": cwdEval, + "relative_path": rel, + }, + ) + } + + return nil +} + +// ValidateSourcePath validates a source directory path for security. +// It ensures the path exists, is a directory, and doesn't contain path traversal attempts. +// +//revive:disable-next-line:function-length +func ValidateSourcePath(path string) error { + if path == "" { + return NewValidationError(CodeValidationRequired, "source path is required") + } + + // Check for path traversal patterns before cleaning + if err := checkPathTraversal(path, "source path"); err != nil { + return err + } + + // Clean and get absolute path + abs, err := cleanAndResolveAbsPath(path, "source path") + if err != nil { + return err + } + cleaned := filepath.Clean(path) + + // Ensure the resolved path is within or below the current working directory for relative paths + if !filepath.IsAbs(path) { + if err := validateWorkingDirectoryBoundary(abs, path); err != nil { + return err + } + } + + // Check if path exists and is a directory + info, err := os.Stat(cleaned) + if err != nil { + if os.IsNotExist(err) { + return NewFileSystemError(CodeFSNotFound, "source directory does not exist").WithFilePath(path) + } + return NewStructuredError( + ErrorTypeFileSystem, + CodeFSAccess, + "cannot access source directory", + path, + map[string]interface{}{ + "error": err.Error(), + }, + ) + } + + if !info.IsDir() { + return NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + "source path must be a directory", + path, + map[string]interface{}{ + "is_file": true, + }, + ) + } + + return nil +} + +// ValidateDestinationPath validates a destination file path for security. +// It ensures the path doesn't contain path traversal attempts and the parent directory exists. +func ValidateDestinationPath(path string) error { + if path == "" { + return NewValidationError(CodeValidationRequired, "destination path is required") + } + + // Check for path traversal patterns before cleaning + if err := checkPathTraversal(path, "destination path"); err != nil { + return err + } + + // Get absolute path to ensure it's not trying to escape current working directory + abs, err := cleanAndResolveAbsPath(path, "destination path") + if err != nil { + return err + } + + // Ensure the destination is not a directory + if info, err := os.Stat(abs); err == nil && info.IsDir() { + return NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + "destination cannot be a directory", + path, + map[string]interface{}{ + "is_directory": true, + }, + ) + } + + // Check if parent directory exists and is writable + parentDir := filepath.Dir(abs) + if parentInfo, err := os.Stat(parentDir); err != nil { + if os.IsNotExist(err) { + return NewStructuredError( + ErrorTypeFileSystem, + CodeFSNotFound, + "destination parent directory does not exist", + path, + map[string]interface{}{ + "parent_dir": parentDir, + }, + ) + } + return NewStructuredError( + ErrorTypeFileSystem, + CodeFSAccess, + "cannot access destination parent directory", + path, + map[string]interface{}{ + "parent_dir": parentDir, + "error": err.Error(), + }, + ) + } else if !parentInfo.IsDir() { + return NewStructuredError( + ErrorTypeValidation, + CodeValidationPath, + "destination parent is not a directory", + path, + map[string]interface{}{ + "parent_dir": parentDir, + }, + ) + } + + return nil +} + +// ValidateConfigPath validates a configuration file path for security. +// It ensures the path doesn't contain path traversal attempts. +func ValidateConfigPath(path string) error { + if path == "" { + return nil // Empty path is allowed for config + } + + // Check for path traversal patterns before cleaning + return checkPathTraversal(path, "config path") +} diff --git a/gibidiutils/paths_additional_test.go b/gibidiutils/paths_additional_test.go new file mode 100644 index 0000000..9799310 --- /dev/null +++ b/gibidiutils/paths_additional_test.go @@ -0,0 +1,368 @@ +package gibidiutils + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetBaseName(t *testing.T) { + tests := []struct { + name string + absPath string + expected string + }{ + { + name: "normal path", + absPath: "/home/user/project", + expected: "project", + }, + { + name: "path with trailing slash", + absPath: "/home/user/project/", + expected: "project", + }, + { + name: "root path", + absPath: "/", + expected: "/", + }, + { + name: "current directory", + absPath: ".", + expected: "output", + }, + { + name: testEmptyPath, + absPath: "", + expected: "output", + }, + { + name: "file path", + absPath: "/home/user/file.txt", + expected: "file.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := GetBaseName(tt.absPath) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestValidateSourcePath(t *testing.T) { + // Create a temp directory for testing + tempDir := t.TempDir() + tempFile := filepath.Join(tempDir, "test.txt") + require.NoError(t, os.WriteFile(tempFile, []byte("test"), 0o600)) + + tests := []struct { + name string + path string + expectedError string + }{ + { + name: testEmptyPath, + path: "", + expectedError: "source path is required", + }, + { + name: testPathTraversalAttempt, + path: "../../../etc/passwd", + expectedError: testPathTraversalDetected, + }, + { + name: "path with double dots", + path: "/home/../etc/passwd", + expectedError: testPathTraversalDetected, + }, + { + name: "non-existent path", + path: "/definitely/does/not/exist", + expectedError: "does not exist", + }, + { + name: "file instead of directory", + path: tempFile, + expectedError: "must be a directory", + }, + { + name: "valid directory", + path: tempDir, + expectedError: "", + }, + { + name: "valid relative path", + path: ".", + expectedError: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateSourcePath(tt.path) + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + + // Check if it's a StructuredError + var structErr *StructuredError + if errors.As(err, &structErr) { + assert.NotEmpty(t, structErr.Code) + assert.NotEqual(t, ErrorTypeUnknown, structErr.Type) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateDestinationPath(t *testing.T) { + tempDir := t.TempDir() + + tests := []struct { + name string + path string + expectedError string + }{ + { + name: testEmptyPath, + path: "", + expectedError: "destination path is required", + }, + { + name: testPathTraversalAttempt, + path: "../../etc/passwd", + expectedError: testPathTraversalDetected, + }, + { + name: "absolute path traversal", + path: "/home/../../../etc/passwd", + expectedError: testPathTraversalDetected, + }, + { + name: "valid new file", + path: filepath.Join(tempDir, "newfile.txt"), + expectedError: "", + }, + { + name: "valid relative path", + path: "output.txt", + expectedError: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateDestinationPath(tt.path) + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateConfigPath(t *testing.T) { + tempDir := t.TempDir() + validConfig := filepath.Join(tempDir, "config.yaml") + require.NoError(t, os.WriteFile(validConfig, []byte("key: value"), 0o600)) + + tests := []struct { + name string + path string + expectedError string + }{ + { + name: testEmptyPath, + path: "", + expectedError: "", // Empty config path is allowed + }, + { + name: testPathTraversalAttempt, + path: "../../../etc/config.yaml", + expectedError: testPathTraversalDetected, + }, + // ValidateConfigPath doesn't check if file exists or is regular file + // It only checks for path traversal + { + name: "valid config file", + path: validConfig, + expectedError: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateConfigPath(tt.path) + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestGetAbsolutePath is already covered in paths_test.go + +func TestValidationErrorTypes(t *testing.T) { + t.Run("source path validation errors", func(t *testing.T) { + // Test empty source + err := ValidateSourcePath("") + assert.Error(t, err) + var structErrEmptyPath *StructuredError + if errors.As(err, &structErrEmptyPath) { + assert.Equal(t, ErrorTypeValidation, structErrEmptyPath.Type) + assert.Equal(t, CodeValidationRequired, structErrEmptyPath.Code) + } + + // Test path traversal + err = ValidateSourcePath("../../../etc") + assert.Error(t, err) + var structErrTraversal *StructuredError + if errors.As(err, &structErrTraversal) { + assert.Equal(t, ErrorTypeValidation, structErrTraversal.Type) + assert.Equal(t, CodeValidationPath, structErrTraversal.Code) + } + }) + + t.Run("destination path validation errors", func(t *testing.T) { + // Test empty destination + err := ValidateDestinationPath("") + assert.Error(t, err) + var structErrEmptyDest *StructuredError + if errors.As(err, &structErrEmptyDest) { + assert.Equal(t, ErrorTypeValidation, structErrEmptyDest.Type) + assert.Equal(t, CodeValidationRequired, structErrEmptyDest.Code) + } + }) + + t.Run("config path validation errors", func(t *testing.T) { + // Test path traversal in config + err := ValidateConfigPath("../../etc/config.yaml") + assert.Error(t, err) + var structErrTraversalInConfig *StructuredError + if errors.As(err, &structErrTraversalInConfig) { + assert.Equal(t, ErrorTypeValidation, structErrTraversalInConfig.Type) + assert.Equal(t, CodeValidationPath, structErrTraversalInConfig.Code) + } + }) +} + +func TestPathSecurityChecks(t *testing.T) { + // Test various path traversal attempts + traversalPaths := []string{ + "../etc/passwd", + "../../root/.ssh/id_rsa", + "/home/../../../etc/shadow", + "./../../sensitive/data", + "foo/../../../bar", + } + + for _, path := range traversalPaths { + t.Run("source_"+path, func(t *testing.T) { + err := ValidateSourcePath(path) + assert.Error(t, err) + assert.Contains(t, err.Error(), testPathTraversal) + }) + + t.Run("dest_"+path, func(t *testing.T) { + err := ValidateDestinationPath(path) + assert.Error(t, err) + assert.Contains(t, err.Error(), testPathTraversal) + }) + + t.Run("config_"+path, func(t *testing.T) { + err := ValidateConfigPath(path) + assert.Error(t, err) + assert.Contains(t, err.Error(), testPathTraversal) + }) + } +} + +func TestSpecialPaths(t *testing.T) { + t.Run("GetBaseName with special paths", func(t *testing.T) { + specialPaths := map[string]string{ + "/": "/", + "": "output", + ".": "output", + "..": "..", + "/.": "output", // filepath.Base("/.") returns "." which matches the output condition + "/..": "..", + "//": "/", + "///": "/", + } + + for path, expected := range specialPaths { + result := GetBaseName(path) + assert.Equal(t, expected, result, "Path: %s", path) + } + }) +} + +func TestPathNormalization(t *testing.T) { + tempDir := t.TempDir() + + t.Run("source path normalization", func(t *testing.T) { + // Create nested directory + nestedDir := filepath.Join(tempDir, "a", "b", "c") + require.NoError(t, os.MkdirAll(nestedDir, 0o750)) + + // Test path with redundant separators + redundantPath := tempDir + string( + os.PathSeparator, + ) + string( + os.PathSeparator, + ) + "a" + string( + os.PathSeparator, + ) + "b" + string( + os.PathSeparator, + ) + "c" + err := ValidateSourcePath(redundantPath) + assert.NoError(t, err) + }) +} + +func TestPathValidationConcurrency(t *testing.T) { + tempDir := t.TempDir() + + // Test concurrent path validation + paths := []string{ + tempDir, + ".", + "/tmp", + } + + errChan := make(chan error, len(paths)*2) + + for _, path := range paths { + go func(p string) { + errChan <- ValidateSourcePath(p) + }(path) + + go func(p string) { + errChan <- ValidateDestinationPath(p + "/output.txt") + }(path) + } + + // Collect results + for i := 0; i < len(paths)*2; i++ { + <-errChan + } + + // No assertions needed - test passes if no panic/race +} diff --git a/utils/paths_test.go b/gibidiutils/paths_test.go similarity index 92% rename from utils/paths_test.go rename to gibidiutils/paths_test.go index fe5b80e..9bc7843 100644 --- a/utils/paths_test.go +++ b/gibidiutils/paths_test.go @@ -1,4 +1,5 @@ -package utils +// Package gibidiutils provides common utility functions for gibidify. +package gibidiutils import ( "os" @@ -138,7 +139,7 @@ func TestGetAbsolutePathSpecialCases(t *testing.T) { target := filepath.Join(tmpDir, "target") link := filepath.Join(tmpDir, "link") - if err := os.Mkdir(target, 0o755); err != nil { + if err := os.Mkdir(target, 0o750); err != nil { t.Fatalf("Failed to create target directory: %v", err) } if err := os.Symlink(target, link); err != nil { @@ -189,7 +190,10 @@ func TestGetAbsolutePathSpecialCases(t *testing.T) { } } -func TestGetAbsolutePathConcurrency(t *testing.T) { +// TestGetAbsolutePathConcurrency verifies that GetAbsolutePath is safe for concurrent use. +// The test intentionally does not use assertions - it will panic if there's a race condition. +// Run with -race flag to detect concurrent access issues. +func TestGetAbsolutePathConcurrency(_ *testing.T) { // Test that GetAbsolutePath is safe for concurrent use paths := []string{".", "..", "test.go", "subdir/file.txt", "/tmp/test"} done := make(chan bool) @@ -224,11 +228,9 @@ func TestGetAbsolutePathErrorFormatting(t *testing.T) { if !strings.Contains(err.Error(), path) { t.Errorf("Error message should contain original path: %v", err) } - } else { + } else if !filepath.IsAbs(got) { // Normal case - just verify we got a valid absolute path - if !filepath.IsAbs(got) { - t.Errorf("Expected absolute path, got: %v", got) - } + t.Errorf("Expected absolute path, got: %v", got) } } diff --git a/gibidiutils/test_constants.go b/gibidiutils/test_constants.go new file mode 100644 index 0000000..3b16302 --- /dev/null +++ b/gibidiutils/test_constants.go @@ -0,0 +1,18 @@ +package gibidiutils + +// Test constants to avoid duplication in test files. +// These constants are used across multiple test files in the gibidiutils package. +const ( + // Error messages + + testErrFileNotFound = "file not found" + testErrWriteFailed = "write failed" + testErrInvalidFormat = "invalid format" + + // Path validation messages + + testEmptyPath = "empty path" + testPathTraversal = "path traversal" + testPathTraversalAttempt = "path traversal attempt" + testPathTraversalDetected = "path traversal attempt detected" +) diff --git a/utils/writers.go b/gibidiutils/writers.go similarity index 78% rename from utils/writers.go rename to gibidiutils/writers.go index f2667bd..f0e5545 100644 --- a/utils/writers.go +++ b/gibidiutils/writers.go @@ -1,8 +1,10 @@ -package utils +// Package gibidiutils provides common utility functions for gibidify. +package gibidiutils import ( "encoding/json" "io" + "math" "strings" ) @@ -34,7 +36,15 @@ func WriteWithErrorWrap(writer io.Writer, content, errorMsg, filePath string) er // StreamContent provides a common streaming implementation with chunk processing. // This eliminates the similar streaming patterns across JSON and Markdown writers. -func StreamContent(reader io.Reader, writer io.Writer, chunkSize int, filePath string, processChunk func([]byte) []byte) error { +// +//revive:disable-next-line:cognitive-complexity +func StreamContent( + reader io.Reader, + writer io.Writer, + chunkSize int, + filePath string, + processChunk func([]byte) []byte, +) error { buf := make([]byte, chunkSize) for { n, err := reader.Read(buf) @@ -55,7 +65,7 @@ func StreamContent(reader io.Reader, writer io.Writer, chunkSize int, filePath s break } if err != nil { - wrappedErr := WrapError(err, ErrorTypeIO, CodeIORead, "failed to read content chunk") + wrappedErr := WrapError(err, ErrorTypeIO, CodeIOFileRead, "failed to read content chunk") if filePath != "" { wrappedErr = wrappedErr.WithFilePath(filePath) } @@ -99,13 +109,27 @@ func EscapeForYAML(content string) string { return content } +// SafeUint64ToInt64WithDefault safely converts uint64 to int64, returning a default value if overflow would occur. +// When defaultValue is 0 (the safe default), clamps to MaxInt64 on overflow to keep guardrails active. +// This prevents overflow from making monitors think memory usage is zero when it's actually maxed out. +func SafeUint64ToInt64WithDefault(value uint64, defaultValue int64) int64 { + if value > math.MaxInt64 { + // When caller uses 0 as "safe" default, clamp to max so overflow still trips guardrails + if defaultValue == 0 { + return math.MaxInt64 + } + return defaultValue + } + return int64(value) //#nosec G115 -- Safe: value <= MaxInt64 checked above +} + // StreamLines provides line-based streaming for YAML content. // This provides an alternative streaming approach for YAML writers. func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProcessor func(string) string) error { // Read all content first (for small files this is fine) content, err := io.ReadAll(reader) if err != nil { - wrappedErr := WrapError(err, ErrorTypeIO, CodeIORead, "failed to read content for line processing") + wrappedErr := WrapError(err, ErrorTypeIO, CodeIOFileRead, "failed to read content for line processing") if filePath != "" { wrappedErr = wrappedErr.WithFilePath(filePath) } @@ -119,13 +143,13 @@ func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProces if lineProcessor != nil { processedLine = lineProcessor(line) } - + // Write line with proper line ending (except for last empty line) lineToWrite := processedLine if i < len(lines)-1 || line != "" { lineToWrite += "\n" } - + if _, writeErr := writer.Write([]byte(lineToWrite)); writeErr != nil { wrappedErr := WrapError(writeErr, ErrorTypeIO, CodeIOWrite, "failed to write processed line") if filePath != "" { @@ -135,4 +159,4 @@ func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProces } } return nil -} \ No newline at end of file +} diff --git a/gibidiutils/writers_test.go b/gibidiutils/writers_test.go new file mode 100644 index 0000000..36724ee --- /dev/null +++ b/gibidiutils/writers_test.go @@ -0,0 +1,111 @@ +// Package gibidiutils provides common utility functions for gibidify. +package gibidiutils + +import ( + "math" + "testing" +) + +func TestSafeUint64ToInt64WithDefault(t *testing.T) { + tests := []struct { + name string + value uint64 + defaultValue int64 + want int64 + }{ + { + name: "normal value within range", + value: 1000, + defaultValue: 0, + want: 1000, + }, + { + name: "zero value", + value: 0, + defaultValue: 0, + want: 0, + }, + { + name: "max int64 exactly", + value: math.MaxInt64, + defaultValue: 0, + want: math.MaxInt64, + }, + { + name: "overflow with zero default clamps to max", + value: math.MaxInt64 + 1, + defaultValue: 0, + want: math.MaxInt64, + }, + { + name: "large overflow with zero default clamps to max", + value: math.MaxUint64, + defaultValue: 0, + want: math.MaxInt64, + }, + { + name: "overflow with custom default returns custom", + value: math.MaxInt64 + 1, + defaultValue: -1, + want: -1, + }, + { + name: "overflow with custom positive default", + value: math.MaxUint64, + defaultValue: 12345, + want: 12345, + }, + { + name: "large value within range", + value: uint64(math.MaxInt64 - 1000), + defaultValue: 0, + want: math.MaxInt64 - 1000, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := SafeUint64ToInt64WithDefault(tt.value, tt.defaultValue) + if got != tt.want { + t.Errorf("SafeUint64ToInt64WithDefault(%d, %d) = %d, want %d", + tt.value, tt.defaultValue, got, tt.want) + } + }) + } +} + +func TestSafeUint64ToInt64WithDefaultGuardrailsBehavior(t *testing.T) { + // Test that overflow with default=0 returns MaxInt64, not 0 + // This is critical for back-pressure and resource monitors + result := SafeUint64ToInt64WithDefault(math.MaxUint64, 0) + if result == 0 { + t.Error("Overflow with default=0 returned 0, which would disable guardrails") + } + if result != math.MaxInt64 { + t.Errorf("Overflow with default=0 should clamp to MaxInt64, got %d", result) + } +} + +// BenchmarkSafeUint64ToInt64WithDefault benchmarks the conversion function +func BenchmarkSafeUint64ToInt64WithDefault(b *testing.B) { + b.Run("normal_value", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = SafeUint64ToInt64WithDefault(1000, 0) + } + }) + + b.Run("overflow_zero_default", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = SafeUint64ToInt64WithDefault(math.MaxUint64, 0) + } + }) + + b.Run("overflow_custom_default", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = SafeUint64ToInt64WithDefault(math.MaxUint64, -1) + } + }) +} diff --git a/go.mod b/go.mod index f95018f..393615a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ivuorinen/gibidify -go 1.24.1 +go 1.25 require ( github.com/fatih/color v1.18.0 @@ -8,26 +8,28 @@ require ( github.com/schollz/progressbar/v3 v3.18.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.28.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.30.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index e58e80d..e668233 100644 --- a/go.sum +++ b/go.sum @@ -7,12 +7,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -21,17 +17,14 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -42,58 +35,37 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= -github.com/sagikazarmark/locafero v0.8.0 h1:mXaMVw7IqxNBxfv3LdWt9MDmcWDQ1fagDH918lOdVaQ= -github.com/sagikazarmark/locafero v0.8.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= -github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= -github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= -github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= -github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= -github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index c768396..62217b9 100644 --- a/main.go +++ b/main.go @@ -23,12 +23,11 @@ func main() { if cli.IsUserError(err) { errorFormatter.FormatError(err) os.Exit(1) - } else { - // System errors still go to logrus for debugging - logrus.Errorf("System error: %v", err) - ui.PrintError("An unexpected error occurred. Please check the logs.") - os.Exit(2) } + // System errors still go to logrus for debugging + logrus.Errorf("System error: %v", err) + ui.PrintError("An unexpected error occurred. Please check the logs.") + os.Exit(2) } } diff --git a/main_test.go b/main_test.go index ab89aac..cb51a5c 100644 --- a/main_test.go +++ b/main_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/ivuorinen/gibidify/config" "github.com/ivuorinen/gibidify/testutil" ) @@ -14,13 +15,22 @@ const ( testFileCount = 1000 ) +// TestMain configures test-time flags for packages. +func TestMain(m *testing.M) { + // Inform packages that we're running under tests so they can adjust noisy logging. + // The config package will suppress the specific info-level message about missing config + // while still allowing tests to enable debug/info level logging when needed. + config.SetRunningInTest(true) + os.Exit(m.Run()) +} + // TestIntegrationFullCLI simulates a full run of the CLI application using adaptive concurrency. func TestIntegrationFullCLI(t *testing.T) { srcDir := setupTestFiles(t) outFilePath := setupOutputFile(t) setupCLIArgs(srcDir, outFilePath) - // Run the application with a background context. + // Run the application with the test context. ctx := t.Context() if runErr := run(ctx); runErr != nil { t.Fatalf("Run failed: %v", runErr) @@ -60,7 +70,7 @@ func setupCLIArgs(srcDir, outFilePath string) { // verifyOutput checks that the output file contains expected content. func verifyOutput(t *testing.T, outFilePath string) { t.Helper() - data, err := os.ReadFile(outFilePath) + data, err := os.ReadFile(outFilePath) // #nosec G304 - test file path is controlled if err != nil { t.Fatalf("Failed to read output file: %v", err) } diff --git a/revive.toml b/revive.toml new file mode 100644 index 0000000..e5cf9de --- /dev/null +++ b/revive.toml @@ -0,0 +1,58 @@ +# revive configuration for gibidify +# See https://revive.run/ for more information + +# Global settings +ignoreGeneratedHeader = false +severity = "warning" +confidence = 0.8 +errorCode = 1 +warningCode = 0 + +# Enable all rules by default, then selectively disable or configure +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +[rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.package-comments] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] +[rule.empty-block] +[rule.superfluous-else] +[rule.unused-parameter] +[rule.unreachable-code] +[rule.redefines-builtin-id] + +# Configure specific rules +[rule.line-length-limit] + arguments = [120] + Exclude = ["**/*_test.go"] + +[rule.function-length] + arguments = [50, 100] + Exclude = ["**/*_test.go"] + +[rule.max-public-structs] + arguments = [10] + +[rule.cognitive-complexity] + arguments = [15] + Exclude = ["**/*_test.go"] + +[rule.cyclomatic] + arguments = [15] + Exclude = ["**/*_test.go"] + +[rule.argument-limit] + arguments = [5] diff --git a/scripts/help.txt b/scripts/help.txt index 1072c99..afb4db0 100644 --- a/scripts/help.txt +++ b/scripts/help.txt @@ -1,25 +1,31 @@ Available targets: - install-tools - Install required linting and development tools - lint - Run all linters (Go, Makefile, shell, YAML) - lint-fix - Run linters with auto-fix enabled - lint-verbose - Run linters with verbose output - test - Run tests - coverage - Run tests with coverage - build - Build the application - clean - Clean build artifacts - all - Run lint, test, and build + install-tools - Install required linting and development tools + lint - Run all linters (Go, EditorConfig, Makefile, shell, YAML) + lint-fix - Run linters with auto-fix enabled + lint-verbose - Run linters with verbose output + test - Run tests + test-coverage - Run tests with coverage output + coverage - Run tests with coverage and generate HTML report + build - Build the application + clean - Clean build artifacts + all - Run lint, test, and build Security targets: - security - Run comprehensive security scan - security-full - Run full security analysis with all tools - vuln-check - Check for dependency vulnerabilities + security - Run comprehensive security scan + security-full - Run full security analysis with all tools + vuln-check - Check for dependency vulnerabilities + +Dependency management: + deps-check - Check for available dependency updates + deps-update - Update all dependencies to latest versions + deps-tidy - Clean up and verify dependencies Benchmark targets: - build-benchmark - Build the benchmark binary - benchmark - Run all benchmarks - benchmark-collection - Run file collection benchmarks - benchmark-processing - Run file processing benchmarks - benchmark-concurrency - Run concurrency benchmarks - benchmark-format - Run format benchmarks + build-benchmark - Build the benchmark binary + benchmark - Run all benchmarks + benchmark-collection - Run file collection benchmarks + benchmark-processing - Run file processing benchmarks + benchmark-concurrency - Run concurrency benchmarks + benchmark-format - Run format benchmarks -Run 'make ' to execute a specific target. \ No newline at end of file +Run 'make ' to execute a specific target. diff --git a/scripts/lint.sh b/scripts/lint.sh index 0070163..a041c40 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,14 +1,23 @@ -#!/bin/bash -set -e +#!/bin/sh +set -eu echo "Running golangci-lint..." golangci-lint run ./... +echo "Running revive..." +revive -config revive.toml -formatter friendly ./... + echo "Running checkmake..." checkmake --config=.checkmake Makefile +echo "Running editorconfig-checker..." +editorconfig-checker + +echo "Running shellcheck..." +shellcheck scripts/*.sh + echo "Running shfmt check..." -shfmt -d . +shfmt -d -i 0 -ci . echo "Running yamllint..." -yamllint -c .yamllint . +yamllint . diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 71c627d..7a8b596 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -1,10 +1,10 @@ -#!/bin/bash -set -euo pipefail +#!/bin/sh +set -eu # Security Scanning Script for gibidify # This script runs comprehensive security checks locally and in CI -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$PROJECT_ROOT" @@ -20,63 +20,177 @@ NC='\033[0m' # No Color # Function to print status print_status() { - echo -e "${BLUE}[INFO]${NC} $1" + printf "${BLUE}[INFO]${NC} %s\n" "$1" } print_warning() { - echo -e "${YELLOW}[WARN]${NC} $1" + printf "${YELLOW}[WARN]${NC} %s\n" "$1" } print_error() { - echo -e "${RED}[ERROR]${NC} $1" + printf "${RED}[ERROR]${NC} %s\n" "$1" } print_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" + printf "${GREEN}[SUCCESS]${NC} %s\n" "$1" +} + +# Run command with timeout if available, otherwise run directly +# Usage: run_with_timeout DURATION COMMAND [ARGS...] +run_with_timeout() { + duration="$1" + shift + + if command -v timeout >/dev/null 2>&1; then + timeout "$duration" "$@" + else + # timeout not available, run command directly + "$@" + fi } # Check if required tools are installed check_dependencies() { print_status "Checking security scanning dependencies..." - local missing_tools=() + missing_tools="" - if ! command -v go &>/dev/null; then - missing_tools+=("go") + if ! command -v go >/dev/null 2>&1; then + missing_tools="${missing_tools}go " + print_error "Go is not installed. Please install Go first." + print_error "Visit https://golang.org/doc/install for installation instructions." + exit 1 fi - if ! command -v golangci-lint &>/dev/null; then + if ! command -v golangci-lint >/dev/null 2>&1; then print_warning "golangci-lint not found, installing..." go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest fi - if ! command -v gosec &>/dev/null; then + if ! command -v gosec >/dev/null 2>&1; then print_warning "gosec not found, installing..." - go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest + go install github.com/securego/gosec/v2/cmd/gosec@latest fi - if ! command -v govulncheck &>/dev/null; then + if ! command -v govulncheck >/dev/null 2>&1; then print_warning "govulncheck not found, installing..." go install golang.org/x/vuln/cmd/govulncheck@latest fi - if ! command -v checkmake &>/dev/null; then + if ! command -v checkmake >/dev/null 2>&1; then print_warning "checkmake not found, installing..." - go install github.com/mrtazz/checkmake/cmd/checkmake@latest + go install github.com/checkmake/checkmake/cmd/checkmake@latest fi - if ! command -v shfmt &>/dev/null; then + if ! command -v shfmt >/dev/null 2>&1; then print_warning "shfmt not found, installing..." go install mvdan.cc/sh/v3/cmd/shfmt@latest fi - if ! command -v yamllint &>/dev/null; then - print_warning "yamllint not found, installing..." - go install github.com/excilsploft/yamllint@latest + if ! command -v yamllint >/dev/null 2>&1; then + print_warning "yamllint not found, attempting to install..." + + # Update PATH to include common user install directories + export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH" + + installed=0 + + # Try pipx first + if command -v pipx >/dev/null 2>&1; then + print_status "Attempting install with pipx..." + if pipx install yamllint; then + # Update PATH to include pipx bin directory + pipx_bin_dir=$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin") + export PATH="$pipx_bin_dir:$PATH" + installed=1 + else + print_warning "pipx install yamllint failed, trying next method..." + fi + fi + + # Try pip3 --user if pipx didn't work + if [ "$installed" -eq 0 ] && command -v pip3 >/dev/null 2>&1; then + print_status "Attempting install with pip3 --user..." + if pip3 install --user yamllint; then + installed=1 + else + print_warning "pip3 install yamllint failed, trying next method..." + fi + fi + + # Try apt-get with smart sudo handling + if [ "$installed" -eq 0 ] && command -v apt-get >/dev/null 2>&1; then + sudo_cmd="" + can_use_apt=false + + # Check if running as root + if [ "$(id -u)" -eq 0 ]; then + print_status "Running as root, no sudo needed for apt-get..." + sudo_cmd="" + can_use_apt=true + elif command -v sudo >/dev/null 2>&1; then + # Try non-interactive sudo first + if sudo -n true 2>/dev/null; then + print_status "Attempting install with apt-get (sudo cached)..." + sudo_cmd="sudo" + can_use_apt=true + elif [ -t 0 ]; then + # TTY available, allow interactive sudo + print_status "Attempting install with apt-get (may prompt for sudo)..." + sudo_cmd="sudo" + can_use_apt=true + else + print_warning "apt-get available but sudo not accessible (non-interactive, no cache). Skipping apt-get." + can_use_apt=false + fi + else + print_warning "apt-get available but sudo not found. Skipping apt-get." + can_use_apt=false + fi + + # Attempt apt-get only if we have permission to use it + if [ "$can_use_apt" = true ]; then + if [ -n "$sudo_cmd" ]; then + if run_with_timeout 300 ${sudo_cmd:+"$sudo_cmd"} apt-get update; then + if run_with_timeout 300 ${sudo_cmd:+"$sudo_cmd"} apt-get install -y yamllint; then + installed=1 + else + print_warning "apt-get install yamllint failed or timed out" + fi + else + print_warning "apt-get update failed or timed out" + fi + else + # Running as root without sudo + if run_with_timeout 300 apt-get update; then + if run_with_timeout 300 apt-get install -y yamllint; then + installed=1 + else + print_warning "apt-get install yamllint failed or timed out" + fi + else + print_warning "apt-get update failed or timed out" + fi + fi + fi + fi + + # Final check with updated PATH + if ! command -v yamllint >/dev/null 2>&1; then + print_error "yamllint installation failed or yamllint still not found in PATH." + print_error "Please install yamllint manually using one of:" + print_error " - pipx install yamllint" + print_error " - pip3 install --user yamllint" + print_error " - sudo apt-get install yamllint (Debian/Ubuntu)" + print_error " - brew install yamllint (macOS)" + exit 1 + fi + + print_status "yamllint successfully installed and found in PATH" fi - if [ ${#missing_tools[@]} -ne 0 ]; then - print_error "Missing required tools: ${missing_tools[*]}" + if [ -n "$missing_tools" ]; then + print_error "Missing required tools: $missing_tools" print_error "Please install the missing tools and try again." exit 1 fi @@ -103,15 +217,41 @@ run_gosec() { run_govulncheck() { print_status "Running govulncheck for dependency vulnerabilities..." - if govulncheck -json ./... >govulncheck-report.json 2>&1; then - print_success "No known vulnerabilities found in dependencies" - else - if grep -q '"finding"' govulncheck-report.json 2>/dev/null; then + # govulncheck with -json always exits 0, so we need to check the output + # Redirect stderr to separate file to avoid corrupting JSON output + govulncheck -json ./... >govulncheck-report.json 2>govulncheck-errors.log + + # Check if there were errors during execution + if [ -s govulncheck-errors.log ]; then + print_warning "govulncheck produced errors (see govulncheck-errors.log)" + fi + + # Use jq to detect finding entries in the JSON output + # govulncheck emits a stream of Message objects, need to slurp and filter for Finding field + if command -v jq >/dev/null 2>&1; then + # First validate JSON is parseable + if ! jq -s '.' govulncheck-report.json >/dev/null 2>&1; then + print_error "govulncheck report contains malformed JSON" + echo "Unable to parse govulncheck-report.json" + return 1 + fi + + # JSON is valid, now check for findings + if jq -s -e 'map(select(.Finding)) | length > 0' govulncheck-report.json >/dev/null 2>&1; then print_error "Vulnerabilities found in dependencies!" echo "Detailed report saved to govulncheck-report.json" return 1 else - print_success "No vulnerabilities found" + print_success "No known vulnerabilities found in dependencies" + fi + else + # Fallback to grep if jq is not available (case-insensitive to match "Finding") + if grep -qi '"finding":' govulncheck-report.json 2>/dev/null; then + print_error "Vulnerabilities found in dependencies!" + echo "Detailed report saved to govulncheck-report.json" + return 1 + else + print_success "No known vulnerabilities found in dependencies" fi fi } @@ -120,7 +260,7 @@ run_govulncheck() { run_security_lint() { print_status "Running security-focused linting..." - local security_linters="gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused,errcheck,ineffassign,staticcheck" + security_linters="gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused,errcheck,ineffassign,staticcheck" if golangci-lint run --enable="$security_linters" --timeout=5m; then print_success "Security linting passed" @@ -134,31 +274,47 @@ run_security_lint() { check_secrets() { print_status "Scanning for potential secrets and sensitive data..." - local secrets_found=false + # POSIX-compatible secrets_found flag using a temp file + secrets_found_file="$(mktemp)" || { + print_error "Failed to create temporary file with mktemp" + exit 1 + } + if [ -z "$secrets_found_file" ]; then + print_error "mktemp returned empty path" + exit 1 + fi + # Clean up temp file on exit and signals (POSIX-portable) + trap 'rm -f "$secrets_found_file"' 0 HUP INT TERM - # Common secret patterns - local patterns=( - "password\s*[:=]\s*['\"][^'\"]{3,}['\"]" - "secret\s*[:=]\s*['\"][^'\"]{3,}['\"]" - "key\s*[:=]\s*['\"][^'\"]{8,}['\"]" - "token\s*[:=]\s*['\"][^'\"]{8,}['\"]" - "api_?key\s*[:=]\s*['\"][^'\"]{8,}['\"]" - "aws_?access_?key" - "aws_?secret" - "AKIA[0-9A-Z]{16}" # AWS Access Key pattern - "github_?token" - "private_?key" - ) - - for pattern in "${patterns[@]}"; do - if grep -r -i -E "$pattern" --include="*.go" . 2>/dev/null; then - print_warning "Potential secret pattern found: $pattern" - secrets_found=true + # Common secret patterns (POSIX [[:space:]] and here-doc quoting) + cat <<'PATTERNS' | while IFS= read -r pattern; do +password[[:space:]]*[:=][[:space:]]*['"][^'"]{3,}['"] +secret[[:space:]]*[:=][[:space:]]*['"][^'"]{3,}['"] +key[[:space:]]*[:=][[:space:]]*['"][^'"]{8,}['"] +token[[:space:]]*[:=][[:space:]]*['"][^'"]{8,}['"] +api_?key[[:space:]]*[:=][[:space:]]*['"][^'"]{8,}['"] +aws_?access_?key +aws_?secret +AKIA[0-9A-Z]{16} +github_?token +private_?key +PATTERNS + if [ -n "$pattern" ]; then + if find . -type f -name "*.go" -exec grep -i -E -H -n -e "$pattern" {} + 2>/dev/null | grep -q .; then + print_warning "Potential secret pattern found: $pattern" + touch "$secrets_found_file" + fi fi done + if [ -f "$secrets_found_file" ]; then + secrets_found=true + else + secrets_found=false + fi + # Check git history for secrets (last 10 commits) - if git log --oneline -10 | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then + if git log --oneline -10 2>/dev/null | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then print_warning "Potential secrets mentioned in recent commit messages" secrets_found=true fi @@ -175,23 +331,23 @@ check_secrets() { check_hardcoded_addresses() { print_status "Checking for hardcoded network addresses..." - local addresses_found=false + addresses_found=false # Look for IP addresses (excluding common safe ones) - if grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . | + if grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . 2>/dev/null | grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" >/dev/null 2>&1; then print_warning "Hardcoded IP addresses found:" - grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . | + grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . 2>/dev/null | grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" || true addresses_found=true fi - # Look for URLs (excluding documentation examples) - if grep -r -E "https?://[^/\s]+" --include="*.go" . | - grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{)" >/dev/null 2>&1; then + # Look for URLs (excluding documentation examples and comments) + if grep -r -E "https?://[^/\s]+" --include="*.go" . 2>/dev/null | + grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{|//.*https?://)" >/dev/null 2>&1; then print_warning "Hardcoded URLs found:" - grep -r -E "https?://[^/\s]+" --include="*.go" . | - grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{)" || true + grep -r -E "https?://[^/\s]+" --include="*.go" . 2>/dev/null | + grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{|//.*https?://)" || true addresses_found=true fi @@ -209,7 +365,7 @@ check_docker_security() { print_status "Checking Docker security..." # Basic Dockerfile security checks - local docker_issues=false + docker_issues=false if grep -q "^USER root" Dockerfile; then print_warning "Dockerfile runs as root user" @@ -221,7 +377,7 @@ check_docker_security() { docker_issues=true fi - if grep -q "RUN.*wget\|RUN.*curl" Dockerfile && ! grep -q "rm.*wget\|rm.*curl" Dockerfile; then + if grep -Eq 'RUN.*(wget|curl)' Dockerfile && ! grep -Eq 'rm.*(wget|curl)' Dockerfile; then print_warning "Dockerfile may leave curl/wget installed" docker_issues=true fi @@ -241,19 +397,21 @@ check_docker_security() { check_file_permissions() { print_status "Checking file permissions..." - local perm_issues=false + perm_issues=false - # Check for overly permissive files - if find . -type f -perm /o+w -not -path "./.git/*" | grep -q .; then + # Check for overly permissive files (using octal for cross-platform compatibility) + # -perm -002 finds files writable by others (works on both BSD and GNU find) + if find . -type f -perm -002 -not -path "./.git/*" 2>/dev/null | grep -q .; then print_warning "World-writable files found:" - find . -type f -perm /o+w -not -path "./.git/*" || true + find . -type f -perm -002 -not -path "./.git/*" 2>/dev/null || true perm_issues=true fi # Check for executable files that shouldn't be - if find . -type f -name "*.go" -perm /a+x | grep -q .; then + # -perm -111 finds files executable by anyone (works on both BSD and GNU find) + if find . -type f -name "*.go" -perm -111 -not -path "./.git/*" 2>/dev/null | grep -q .; then print_warning "Executable Go files found (should not be executable):" - find . -type f -name "*.go" -perm /a+x || true + find . -type f -name "*.go" -perm -111 -not -path "./.git/*" 2>/dev/null || true perm_issues=true fi @@ -285,7 +443,7 @@ check_makefile() { check_shell_scripts() { print_status "Checking shell script formatting..." - if find . -name "*.sh" -type f | head -1 | grep -q .; then + if find . -name "*.sh" -type f 2>/dev/null | head -1 | grep -q .; then if shfmt -d .; then print_success "Shell script formatting check passed" else @@ -301,8 +459,8 @@ check_shell_scripts() { check_yaml_files() { print_status "Checking YAML files..." - if find . -name "*.yml" -o -name "*.yaml" -type f | head -1 | grep -q .; then - if yamllint -c .yamllint .; then + if find . \( -name "*.yml" -o -name "*.yaml" \) -type f 2>/dev/null | head -1 | grep -q .; then + if yamllint .; then print_success "YAML files check passed" else print_error "YAML file issues detected!" @@ -317,7 +475,7 @@ check_yaml_files() { generate_report() { print_status "Generating security scan report..." - local report_file="security-report.md" + report_file="security-report.md" cat >"$report_file" <