mirror of
https://github.com/ivuorinen/gibidify.git
synced 2026-01-26 03:24:05 +00:00
feat: update go to 1.25, add permissions and envs (#49)
* chore(ci): update go to 1.25, add permissions and envs * fix(ci): update pr-lint.yml * chore: update go, fix linting * fix: tests and linting * fix(lint): lint fixes, renovate should now pass * fix: updates, security upgrades * chore: workflow updates, lint * fix: more lint, checkmake, and other fixes * fix: more lint, convert scripts to POSIX compliant * fix: simplify codeql workflow * tests: increase test coverage, fix found issues * fix(lint): editorconfig checking, add to linters * fix(lint): shellcheck, add to linters * fix(lint): apply cr comment suggestions * fix(ci): remove step-security/harden-runner * fix(lint): remove duplication, apply cr fixes * fix(ci): tests in CI/CD pipeline * chore(lint): deduplication of strings * fix(lint): apply cr comment suggestions * fix(ci): actionlint * fix(lint): apply cr comment suggestions * chore: lint, add deps management
This commit is contained in:
10
.checkmake
10
.checkmake
@@ -1,8 +1,14 @@
|
|||||||
# checkmake configuration
|
# checkmake configuration
|
||||||
# See: https://github.com/mrtazz/checkmake#configuration
|
# See: https://github.com/checkmake/checkmake#configuration
|
||||||
|
|
||||||
[rules.timestampexpansion]
|
[rules.timestampexpansion]
|
||||||
disabled = true
|
disabled = true
|
||||||
|
|
||||||
[rules.maxbodylength]
|
[rules.maxbodylength]
|
||||||
disabled = true
|
disabled = true
|
||||||
|
|
||||||
|
[rules.minphony]
|
||||||
|
disabled = true
|
||||||
|
|
||||||
|
[rules.phonydeclared]
|
||||||
|
disabled = true
|
||||||
|
|||||||
38
.dockerignore
Normal file
38
.dockerignore
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
# Git
|
||||||
|
.git
|
||||||
|
.github
|
||||||
|
.gitignore
|
||||||
|
|
||||||
|
# Build artifacts
|
||||||
|
gibidify
|
||||||
|
gibidify-*
|
||||||
|
dist/
|
||||||
|
coverage.out
|
||||||
|
coverage.html
|
||||||
|
test-results.json
|
||||||
|
*.sarif
|
||||||
|
|
||||||
|
# Documentation
|
||||||
|
*.md
|
||||||
|
docs/
|
||||||
|
|
||||||
|
# Config and tooling
|
||||||
|
.checkmake
|
||||||
|
.editorconfig
|
||||||
|
.golangci.yml
|
||||||
|
.yamllint
|
||||||
|
revive.toml
|
||||||
|
|
||||||
|
# Scripts
|
||||||
|
scripts/
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode
|
||||||
|
.idea
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
@@ -8,19 +8,26 @@ indent_size = 2
|
|||||||
indent_style = tab
|
indent_style = tab
|
||||||
tab_width = 2
|
tab_width = 2
|
||||||
|
|
||||||
[*.yml]
|
[*.go]
|
||||||
indent_style = space
|
max_line_length = 120
|
||||||
|
|
||||||
[*.md]
|
[*.md]
|
||||||
trim_trailing_whitespace = false
|
trim_trailing_whitespace = false
|
||||||
|
|
||||||
[*.{yml,yaml,json}]
|
[*.{yml,yaml,json,toml}]
|
||||||
indent_style = space
|
indent_style = space
|
||||||
max_line_length = 250
|
max_line_length = 250
|
||||||
|
|
||||||
|
[*.{yaml.example,yml.example}]
|
||||||
|
indent_style = space
|
||||||
|
|
||||||
|
[.yamllint]
|
||||||
|
indent_style = space
|
||||||
|
|
||||||
[LICENSE]
|
[LICENSE]
|
||||||
max_line_length = 80
|
max_line_length = 80
|
||||||
indent_size = 0
|
indent_size = 0
|
||||||
indent_style = space
|
indent_style = space
|
||||||
trim_trailing_whitespace = true
|
|
||||||
|
|
||||||
|
[Makefile]
|
||||||
|
max_line_length = 80
|
||||||
|
|||||||
14
.editorconfig-checker.json
Normal file
14
.editorconfig-checker.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"Exclude": [".git", "vendor", "node_modules", "README\\.md"],
|
||||||
|
"AllowedContentTypes": [],
|
||||||
|
"PassedFiles": [],
|
||||||
|
"Disable": {
|
||||||
|
"IndentSize": false,
|
||||||
|
"EndOfLine": false,
|
||||||
|
"InsertFinalNewline": false,
|
||||||
|
"TrimTrailingWhitespace": false,
|
||||||
|
"MaxLineLength": false
|
||||||
|
},
|
||||||
|
"SpacesAfterTabs": false,
|
||||||
|
"NoColor": false
|
||||||
|
}
|
||||||
15
.github/actions/setup/action.yml
vendored
Normal file
15
.github/actions/setup/action.yml
vendored
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
name: "Setup Go with Runner Hardening"
|
||||||
|
description: "Reusable action to set up Go"
|
||||||
|
inputs:
|
||||||
|
token:
|
||||||
|
description: "GitHub token for checkout (optional)"
|
||||||
|
required: false
|
||||||
|
default: ""
|
||||||
|
runs:
|
||||||
|
using: "composite"
|
||||||
|
steps:
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
|
||||||
|
with:
|
||||||
|
go-version-file: "go.mod"
|
||||||
|
cache: true
|
||||||
90
.github/workflows/build-test-publish.yml
vendored
90
.github/workflows/build-test-publish.yml
vendored
@@ -9,8 +9,7 @@ on:
|
|||||||
release:
|
release:
|
||||||
types: [created]
|
types: [created]
|
||||||
|
|
||||||
permissions:
|
permissions: {}
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
@@ -25,51 +24,60 @@ jobs:
|
|||||||
statuses: write
|
statuses: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Harden the runner (Audit all outbound calls)
|
- name: Checkout repository
|
||||||
uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1
|
|
||||||
with:
|
|
||||||
egress-policy: audit
|
|
||||||
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
|
uses: ./.github/actions/setup
|
||||||
with:
|
with:
|
||||||
go-version-file: "./go.mod"
|
token: ${{ github.token }}
|
||||||
cache: true
|
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Download dependencies
|
||||||
run: go mod tidy
|
shell: bash
|
||||||
|
run: go mod download
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests with coverage
|
||||||
run: go test -json ./... > test-results.json
|
shell: bash
|
||||||
|
run: |
|
||||||
- name: Generate coverage report
|
go test -race -covermode=atomic -json -coverprofile=coverage.out ./... | tee test-results.json
|
||||||
run: go test -coverprofile=coverage.out ./...
|
|
||||||
|
|
||||||
- name: Check coverage
|
- name: Check coverage
|
||||||
id: coverage
|
id: coverage
|
||||||
|
if: always()
|
||||||
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
|
if [[ ! -f coverage.out ]]; then
|
||||||
|
echo "coverage.out is missing; tests likely failed before producing coverage"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
coverage="$(go tool cover -func=coverage.out | grep total | awk '{print substr($3, 1, length($3)-1)}')"
|
coverage="$(go tool cover -func=coverage.out | grep total | awk '{print substr($3, 1, length($3)-1)}')"
|
||||||
echo "total_coverage=$coverage" >> "$GITHUB_ENV"
|
echo "total_coverage=$coverage" >> "$GITHUB_ENV"
|
||||||
echo "Coverage: $coverage%"
|
echo "Coverage: $coverage%"
|
||||||
|
|
||||||
- name: Upload test results
|
- name: Upload test results
|
||||||
|
if: always()
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: test-results
|
name: test-results
|
||||||
path: test-results.json
|
path: test-results.json
|
||||||
|
|
||||||
- name: Cleanup
|
- name: Cleanup
|
||||||
run: rm coverage.out
|
if: always()
|
||||||
|
shell: bash
|
||||||
|
run: rm -f coverage.out test-results.json
|
||||||
|
|
||||||
- name: Fail if coverage is below threshold
|
- name: Fail if coverage is below threshold
|
||||||
|
if: always()
|
||||||
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
if (( $(echo "$total_coverage < 50" | bc -l) )); then
|
if [[ -z "${total_coverage:-}" ]]; then
|
||||||
echo "Coverage ($total_coverage%) is below the threshold (50%)"
|
echo "total_coverage is unset; previous step likely failed"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
awk -v cov="$total_coverage" 'BEGIN{ if (cov < 60) exit 1; else exit 0 }' || {
|
||||||
|
echo "Coverage ($total_coverage%) is below the threshold (60%)"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
build:
|
build:
|
||||||
name: Build Binaries
|
name: Build Binaries
|
||||||
@@ -89,13 +97,13 @@ jobs:
|
|||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
|
uses: ./.github/actions/setup
|
||||||
with:
|
with:
|
||||||
go-version-file: "./go.mod"
|
token: ${{ github.token }}
|
||||||
|
|
||||||
- name: Run go mod tidy
|
- name: Download dependencies
|
||||||
run: go mod tidy
|
run: go mod download
|
||||||
|
|
||||||
- name: Build binary for ${{ matrix.goos }}-${{ matrix.goarch }}
|
- name: Build binary for ${{ matrix.goos }}-${{ matrix.goarch }}
|
||||||
run: |
|
run: |
|
||||||
@@ -132,24 +140,24 @@ jobs:
|
|||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||||
|
|
||||||
- name: Download Linux binaries
|
- name: Setup Go
|
||||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
uses: ./.github/actions/setup
|
||||||
with:
|
with:
|
||||||
name: gibidify-linux-amd64
|
token: ${{ github.token }}
|
||||||
path: .
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||||
|
|
||||||
- name: Log in to GitHub Container Registry
|
- name: Log in to GitHub Container Registry
|
||||||
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
|
|
||||||
|
|
||||||
- name: Build and push multi-arch Docker image
|
|
||||||
run: |
|
run: |
|
||||||
chmod +x gibidify-linux-amd64
|
echo "${{ github.token }}" | docker login ghcr.io \
|
||||||
mv gibidify-linux-amd64 gibidify
|
-u "$(echo "${{ github.actor }}" | tr '[:upper:]' '[:lower:]')" \
|
||||||
docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 \
|
--password-stdin
|
||||||
--tag ghcr.io/${{ github.repository }}/gibidify:${{ github.ref_name }} \
|
|
||||||
--tag ghcr.io/${{ github.repository }}/gibidify:latest \
|
- name: Build and push Docker image
|
||||||
--push \
|
run: |
|
||||||
--squash .
|
repo="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
docker buildx build --platform linux/amd64 \
|
||||||
|
--tag "ghcr.io/${repo}/gibidify:${{ github.ref_name }}" \
|
||||||
|
--tag "ghcr.io/${repo}/gibidify:latest" \
|
||||||
|
--push .
|
||||||
|
|||||||
39
.github/workflows/codeql.yml
vendored
Normal file
39
.github/workflows/codeql.yml
vendored
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
name: CodeQL Analysis
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, develop]
|
||||||
|
pull_request:
|
||||||
|
branches: [main, develop]
|
||||||
|
|
||||||
|
permissions: {}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
analyze:
|
||||||
|
name: Analyze Code
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
security-events: write
|
||||||
|
contents: read
|
||||||
|
actions: read
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||||
|
|
||||||
|
- name: Setup Go
|
||||||
|
uses: ./.github/actions/setup
|
||||||
|
with:
|
||||||
|
token: ${{ github.token }}
|
||||||
|
|
||||||
|
- name: Initialize CodeQL
|
||||||
|
uses: github/codeql-action/init@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7
|
||||||
|
with:
|
||||||
|
languages: go
|
||||||
|
|
||||||
|
- name: Autobuild
|
||||||
|
uses: github/codeql-action/autobuild@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7
|
||||||
|
|
||||||
|
- name: Perform CodeQL Analysis
|
||||||
|
uses: github/codeql-action/analyze@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7
|
||||||
11
.github/workflows/pr-lint.yml
vendored
11
.github/workflows/pr-lint.yml
vendored
@@ -9,7 +9,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches: [master, main]
|
branches: [master, main]
|
||||||
|
|
||||||
permissions: read-all
|
permissions: {}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
Linter:
|
Linter:
|
||||||
@@ -21,7 +21,12 @@ jobs:
|
|||||||
pull-requests: write
|
pull-requests: write
|
||||||
statuses: write
|
statuses: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||||
|
|
||||||
|
- name: Setup Go
|
||||||
|
uses: ./.github/actions/setup
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
token: ${{ github.token }}
|
||||||
|
|
||||||
- uses: ivuorinen/actions/pr-lint@dc895c40ffdce61ab057fb992f4e00f1efdcbcbf # 25.10.7
|
- uses: ivuorinen/actions/pr-lint@dc895c40ffdce61ab057fb992f4e00f1efdcbcbf # 25.10.7
|
||||||
|
|||||||
86
.github/workflows/security.yml
vendored
86
.github/workflows/security.yml
vendored
@@ -7,45 +7,37 @@ on:
|
|||||||
branches: [main, develop]
|
branches: [main, develop]
|
||||||
schedule:
|
schedule:
|
||||||
# Run security scan weekly on Sundays at 00:00 UTC
|
# Run security scan weekly on Sundays at 00:00 UTC
|
||||||
- cron: '0 0 * * 0'
|
- cron: "0 0 * * 0"
|
||||||
|
|
||||||
permissions:
|
permissions: {}
|
||||||
security-events: write
|
|
||||||
contents: read
|
|
||||||
actions: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
security:
|
security:
|
||||||
name: Security Analysis
|
name: Security Analysis
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
security-events: write
|
||||||
|
contents: read
|
||||||
|
actions: read
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout repository
|
||||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||||
|
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6
|
uses: ./.github/actions/setup
|
||||||
with:
|
with:
|
||||||
go-version: '1.23'
|
token: ${{ github.token }}
|
||||||
|
|
||||||
- name: Cache Go modules
|
|
||||||
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4
|
|
||||||
with:
|
|
||||||
path: |
|
|
||||||
~/.cache/go-build
|
|
||||||
~/go/pkg/mod
|
|
||||||
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-go-
|
|
||||||
|
|
||||||
# Security Scanning with gosec
|
# Security Scanning with gosec
|
||||||
- name: Run gosec Security Scanner
|
- name: Run gosec Security Scanner
|
||||||
uses: securego/gosec@15d5c61e866bc2e2e8389376a31f1e5e09bde7d8 # v2.22.9
|
uses: securego/gosec@15d5c61e866bc2e2e8389376a31f1e5e09bde7d8 # v2.22.9
|
||||||
with:
|
with:
|
||||||
args: '-fmt sarif -out gosec-results.sarif ./...'
|
args: "-fmt sarif -out gosec-results.sarif ./..."
|
||||||
|
|
||||||
- name: Upload gosec results to GitHub Security tab
|
- name: Upload gosec results to GitHub Security tab
|
||||||
uses: github/codeql-action/upload-sarif@df559355d593797519d70b90fc8edd5db049e7a2 # v3
|
uses: github/codeql-action/upload-sarif@e296a935590eb16afc0c0108289f68c87e2a89a5 # v4.30.7
|
||||||
if: always()
|
if: always()
|
||||||
with:
|
with:
|
||||||
sarif_file: gosec-results.sarif
|
sarif_file: gosec-results.sarif
|
||||||
@@ -60,24 +52,17 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
if [ -s govulncheck-results.json ]; then
|
if [ -s govulncheck-results.json ]; then
|
||||||
echo "::warning::Vulnerability check completed. Check govulncheck-results.json for details."
|
echo "::warning::Vulnerability check completed. Check govulncheck-results.json for details."
|
||||||
if grep -q '"finding"' govulncheck-results.json; then
|
if grep -i -q '"finding"' govulncheck-results.json; then
|
||||||
echo "::error::Vulnerabilities found in dependencies!"
|
echo "::error::Vulnerabilities found in dependencies!"
|
||||||
cat govulncheck-results.json
|
cat govulncheck-results.json
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Additional Security Linting
|
|
||||||
- name: Run security-focused golangci-lint
|
|
||||||
run: |
|
|
||||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
|
||||||
golangci-lint run --enable=gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused \
|
|
||||||
--timeout=5m
|
|
||||||
|
|
||||||
# Makefile Linting
|
# Makefile Linting
|
||||||
- name: Run checkmake on Makefile
|
- name: Run checkmake on Makefile
|
||||||
run: |
|
run: |
|
||||||
go install github.com/mrtazz/checkmake/cmd/checkmake@latest
|
go install github.com/checkmake/checkmake/cmd/checkmake@latest
|
||||||
checkmake --config=.checkmake Makefile
|
checkmake --config=.checkmake Makefile
|
||||||
|
|
||||||
# Shell Script Formatting Check
|
# Shell Script Formatting Check
|
||||||
@@ -86,27 +71,11 @@ jobs:
|
|||||||
go install mvdan.cc/sh/v3/cmd/shfmt@latest
|
go install mvdan.cc/sh/v3/cmd/shfmt@latest
|
||||||
shfmt -d .
|
shfmt -d .
|
||||||
|
|
||||||
# YAML Linting
|
|
||||||
- name: Run YAML linting
|
- name: Run YAML linting
|
||||||
run: |
|
uses: ibiqlik/action-yamllint@2576378a8e339169678f9939646ee3ee325e845c # v3.1.1
|
||||||
go install github.com/excilsploft/yamllint@latest
|
with:
|
||||||
yamllint -c .yamllint .
|
file_or_dir: .
|
||||||
|
strict: true
|
||||||
# Secrets Detection (basic patterns)
|
|
||||||
- name: Run secrets detection
|
|
||||||
run: |
|
|
||||||
echo "Scanning for potential secrets..."
|
|
||||||
# Look for common secret patterns
|
|
||||||
git log --all --full-history -- . | grep -i -E "(password|secret|key|token|api_key)" || true
|
|
||||||
find . -type f -name "*.go" -exec grep -H -i -E "(password|secret|key|token|api_key)\s*[:=]" {} \; || true
|
|
||||||
|
|
||||||
# Check for hardcoded IPs and URLs
|
|
||||||
- name: Check for hardcoded network addresses
|
|
||||||
run: |
|
|
||||||
echo "Scanning for hardcoded network addresses..."
|
|
||||||
find . -type f -name "*.go" -exec grep -H -E "([0-9]{1,3}\.){3}[0-9]{1,3}" {} \; || true
|
|
||||||
find . -type f -name "*.go" -exec grep -H -E "https?://[^/\s]+" {} \; | \
|
|
||||||
grep -v "example.com|localhost|127.0.0.1" || true
|
|
||||||
|
|
||||||
# Docker Security (if Dockerfile exists)
|
# Docker Security (if Dockerfile exists)
|
||||||
- name: Run Docker security scan
|
- name: Run Docker security scan
|
||||||
@@ -115,24 +84,9 @@ jobs:
|
|||||||
docker run --rm -v "$PWD":/workspace \
|
docker run --rm -v "$PWD":/workspace \
|
||||||
aquasec/trivy:latest fs --security-checks vuln,config /workspace/Dockerfile || true
|
aquasec/trivy:latest fs --security-checks vuln,config /workspace/Dockerfile || true
|
||||||
|
|
||||||
# SAST with CodeQL (if available)
|
|
||||||
- name: Initialize CodeQL
|
|
||||||
if: github.event_name != 'schedule'
|
|
||||||
uses: github/codeql-action/init@df559355d593797519d70b90fc8edd5db049e7a2 # v3
|
|
||||||
with:
|
|
||||||
languages: go
|
|
||||||
|
|
||||||
- name: Autobuild
|
|
||||||
if: github.event_name != 'schedule'
|
|
||||||
uses: github/codeql-action/autobuild@df559355d593797519d70b90fc8edd5db049e7a2 # v3
|
|
||||||
|
|
||||||
- name: Perform CodeQL Analysis
|
|
||||||
if: github.event_name != 'schedule'
|
|
||||||
uses: github/codeql-action/analyze@df559355d593797519d70b90fc8edd5db049e7a2 # v3
|
|
||||||
|
|
||||||
# Upload artifacts for review
|
# Upload artifacts for review
|
||||||
- name: Upload security scan results
|
- name: Upload security scan results
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
if: always()
|
if: always()
|
||||||
with:
|
with:
|
||||||
name: security-scan-results
|
name: security-scan-results
|
||||||
|
|||||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -12,3 +12,8 @@ megalinter-reports/*
|
|||||||
coverage.*
|
coverage.*
|
||||||
*.out
|
*.out
|
||||||
gibidify-benchmark
|
gibidify-benchmark
|
||||||
|
gosec-report.json
|
||||||
|
gosec-results.sarif
|
||||||
|
govulncheck-report.json
|
||||||
|
govulncheck-errors.log
|
||||||
|
security-report.md
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
1.23.0
|
1.25.1
|
||||||
|
|||||||
@@ -15,6 +15,9 @@ PRINT_ALPACA: false # Print Alpaca logo in console
|
|||||||
SARIF_REPORTER: true # Generate SARIF report
|
SARIF_REPORTER: true # Generate SARIF report
|
||||||
SHOW_SKIPPED_LINTERS: false # Show skipped linters in MegaLinter log
|
SHOW_SKIPPED_LINTERS: false # Show skipped linters in MegaLinter log
|
||||||
|
|
||||||
|
GO_REVIVE_CLI_LINT_MODE: project
|
||||||
|
|
||||||
DISABLE_LINTERS:
|
DISABLE_LINTERS:
|
||||||
- REPOSITORY_DEVSKIM
|
- REPOSITORY_DEVSKIM
|
||||||
- REPOSITORY_TRIVY
|
- REPOSITORY_TRIVY
|
||||||
|
- GO_GOLANGCI_LINT
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ repos:
|
|||||||
- id: golangci-lint
|
- id: golangci-lint
|
||||||
args: ["--timeout=5m"]
|
args: ["--timeout=5m"]
|
||||||
- repo: https://github.com/tekwizely/pre-commit-golang
|
- repo: https://github.com/tekwizely/pre-commit-golang
|
||||||
rev: v1.0.0-rc.1
|
rev: v1.0.0-rc.2
|
||||||
hooks:
|
hooks:
|
||||||
- id: go-build-mod
|
- id: go-build-mod
|
||||||
alias: build
|
alias: build
|
||||||
@@ -13,3 +13,12 @@ repos:
|
|||||||
alias: tidy
|
alias: tidy
|
||||||
- id: go-fmt
|
- id: go-fmt
|
||||||
alias: fmt
|
alias: fmt
|
||||||
|
- repo: https://github.com/editorconfig-checker/editorconfig-checker.python
|
||||||
|
rev: 3.4.0
|
||||||
|
hooks:
|
||||||
|
- id: editorconfig-checker
|
||||||
|
alias: ec
|
||||||
|
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||||
|
rev: v0.11.0.1
|
||||||
|
hooks:
|
||||||
|
- id: shellcheck
|
||||||
|
|||||||
1
.serena/.gitignore
vendored
Normal file
1
.serena/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
/cache
|
||||||
73
.serena/project.yml
Normal file
73
.serena/project.yml
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
|
||||||
|
# * For C, use cpp
|
||||||
|
# * For JavaScript, use typescript
|
||||||
|
# Special requirements:
|
||||||
|
# * csharp: Requires the presence of a .sln file in the project folder.
|
||||||
|
language: go
|
||||||
|
|
||||||
|
# whether to use the project's gitignore file to ignore files
|
||||||
|
# Added on 2025-04-07
|
||||||
|
ignore_all_files_in_gitignore: true
|
||||||
|
# list of additional paths to ignore
|
||||||
|
# same syntax as gitignore, so you can use * and **
|
||||||
|
# Was previously called `ignored_dirs`, please update your config if you are using that.
|
||||||
|
# Added (renamed) on 2025-04-07
|
||||||
|
ignored_paths: []
|
||||||
|
|
||||||
|
# whether the project is in read-only mode
|
||||||
|
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||||
|
# Added on 2025-04-18
|
||||||
|
read_only: false
|
||||||
|
|
||||||
|
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||||
|
# Below is the complete list of tools for convenience.
|
||||||
|
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||||
|
# execute `uv run scripts/print_tool_overview.py`.
|
||||||
|
#
|
||||||
|
# * `activate_project`: Activates a project by name.
|
||||||
|
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||||
|
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||||
|
# * `delete_lines`: Deletes a range of lines within a file.
|
||||||
|
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||||
|
# * `execute_shell_command`: Executes a shell command.
|
||||||
|
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||||
|
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location
|
||||||
|
# (optionally filtered by type).
|
||||||
|
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given
|
||||||
|
# name/substring (optionally filtered by type).
|
||||||
|
# * `get_current_config`: Prints the current configuration of the agent, including the active
|
||||||
|
# and available projects, tools, contexts, and modes.
|
||||||
|
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||||
|
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||||
|
# Should only be used in settings where the system prompt cannot be set,
|
||||||
|
# e.g. in clients you have no control over, like Claude Desktop.
|
||||||
|
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||||
|
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||||
|
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||||
|
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||||
|
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||||
|
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks,
|
||||||
|
# e.g. for testing or building).
|
||||||
|
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation
|
||||||
|
# (in order to continue with the necessary context).
|
||||||
|
# * `read_file`: Reads a file within the project directory.
|
||||||
|
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||||
|
# * `remove_project`: Removes a project from the Serena configuration.
|
||||||
|
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||||
|
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||||
|
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||||
|
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||||
|
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||||
|
# * `switch_modes`: Activates modes by providing a list of their names
|
||||||
|
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||||
|
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on
|
||||||
|
# track with the current task.
|
||||||
|
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||||
|
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||||
|
excluded_tools: []
|
||||||
|
|
||||||
|
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||||
|
# (contrary to the memories, which are loaded on demand).
|
||||||
|
initial_prompt: ""
|
||||||
|
|
||||||
|
project_name: "gibidify"
|
||||||
@@ -44,4 +44,7 @@ EditorConfig (LF, tabs), semantic commits, testing required
|
|||||||
|
|
||||||
## Workflow
|
## Workflow
|
||||||
|
|
||||||
1. `make lint-fix` first 2. >80% coverage 3. Follow patterns 4. Update docs
|
1. `make lint-fix` first
|
||||||
|
2. >80% coverage
|
||||||
|
3. Follow patterns
|
||||||
|
4. Update docs
|
||||||
|
|||||||
41
Dockerfile
41
Dockerfile
@@ -1,17 +1,38 @@
|
|||||||
# Use a minimal base image
|
# Build stage - builds the binary for the target architecture
|
||||||
|
FROM --platform=$BUILDPLATFORM golang:1.25.1-alpine AS builder
|
||||||
|
|
||||||
|
# Build arguments automatically set by buildx
|
||||||
|
ARG TARGETOS
|
||||||
|
ARG TARGETARCH
|
||||||
|
ARG TARGETVARIANT
|
||||||
|
|
||||||
|
WORKDIR /build
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Build the binary for the target platform
|
||||||
|
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||||
|
go build -ldflags="-s -w" -o gibidify .
|
||||||
|
|
||||||
|
# Runtime stage - minimal image with the binary
|
||||||
FROM alpine:3.22.1
|
FROM alpine:3.22.1
|
||||||
|
|
||||||
# Add user
|
# Install ca-certificates for HTTPS and create non-root user
|
||||||
RUN useradd -ms /bin/bash gibidify
|
# hadolint ignore=DL3018
|
||||||
|
# kics-scan ignore-line
|
||||||
|
RUN apk add --no-cache ca-certificates && \
|
||||||
|
adduser -D -s /bin/sh gibidify
|
||||||
|
|
||||||
# Use the new user
|
# Copy the binary from builder
|
||||||
|
COPY --from=builder /build/gibidify /usr/local/bin/gibidify
|
||||||
|
|
||||||
|
# Use non-root user
|
||||||
USER gibidify
|
USER gibidify
|
||||||
|
|
||||||
# Copy the gibidify binary into the container
|
|
||||||
COPY gibidify /usr/local/bin/gibidify
|
|
||||||
|
|
||||||
# Ensure the binary is executable
|
|
||||||
RUN chmod +x /usr/local/bin/gibidify
|
|
||||||
|
|
||||||
# Set the entrypoint
|
# Set the entrypoint
|
||||||
ENTRYPOINT ["/usr/local/bin/gibidify"]
|
ENTRYPOINT ["/usr/local/bin/gibidify"]
|
||||||
|
|||||||
75
Makefile
75
Makefile
@@ -1,4 +1,8 @@
|
|||||||
.PHONY: help install-tools lint lint-fix lint-verbose test coverage build clean all build-benchmark benchmark benchmark-collection benchmark-processing benchmark-concurrency benchmark-format security security-full vuln-check check-all dev-setup
|
.PHONY: all clean test test-coverage build coverage help lint lint-fix \
|
||||||
|
lint-verbose install-tools benchmark benchmark-collection \
|
||||||
|
benchmark-concurrency benchmark-format benchmark-processing \
|
||||||
|
build-benchmark check-all ci-lint ci-test dev-setup security \
|
||||||
|
security-full vuln-check deps-update deps-check deps-tidy
|
||||||
|
|
||||||
# Default target shows help
|
# Default target shows help
|
||||||
.DEFAULT_GOAL := help
|
.DEFAULT_GOAL := help
|
||||||
@@ -6,7 +10,7 @@
|
|||||||
# All target runs full workflow
|
# All target runs full workflow
|
||||||
all: lint test build
|
all: lint test build
|
||||||
|
|
||||||
# Help target
|
# Help target
|
||||||
help:
|
help:
|
||||||
@cat scripts/help.txt
|
@cat scripts/help.txt
|
||||||
|
|
||||||
@@ -16,6 +20,8 @@ install-tools:
|
|||||||
@go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
@go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||||
@echo "Installing gofumpt..."
|
@echo "Installing gofumpt..."
|
||||||
@go install mvdan.cc/gofumpt@latest
|
@go install mvdan.cc/gofumpt@latest
|
||||||
|
@echo "Installing golines..."
|
||||||
|
@go install github.com/segmentio/golines@latest
|
||||||
@echo "Installing goimports..."
|
@echo "Installing goimports..."
|
||||||
@go install golang.org/x/tools/cmd/goimports@latest
|
@go install golang.org/x/tools/cmd/goimports@latest
|
||||||
@echo "Installing staticcheck..."
|
@echo "Installing staticcheck..."
|
||||||
@@ -24,12 +30,19 @@ install-tools:
|
|||||||
@go install github.com/securego/gosec/v2/cmd/gosec@latest
|
@go install github.com/securego/gosec/v2/cmd/gosec@latest
|
||||||
@echo "Installing gocyclo..."
|
@echo "Installing gocyclo..."
|
||||||
@go install github.com/fzipp/gocyclo/cmd/gocyclo@latest
|
@go install github.com/fzipp/gocyclo/cmd/gocyclo@latest
|
||||||
|
@echo "Installing revive..."
|
||||||
|
@go install github.com/mgechev/revive@latest
|
||||||
@echo "Installing checkmake..."
|
@echo "Installing checkmake..."
|
||||||
@go install github.com/mrtazz/checkmake/cmd/checkmake@latest
|
@go install github.com/checkmake/checkmake/cmd/checkmake@latest
|
||||||
|
@echo "Installing shellcheck..."
|
||||||
|
@go install github.com/koalaman/shellcheck/cmd/shellcheck@latest
|
||||||
@echo "Installing shfmt..."
|
@echo "Installing shfmt..."
|
||||||
@go install mvdan.cc/sh/v3/cmd/shfmt@latest
|
@go install mvdan.cc/sh/v3/cmd/shfmt@latest
|
||||||
@echo "Installing yamllint (Go-based)..."
|
@echo "Installing yamllint (Go-based)..."
|
||||||
@go install github.com/excilsploft/yamllint@latest
|
@go install github.com/excilsploft/yamllint@latest
|
||||||
|
@echo "Installing editorconfig-checker..."
|
||||||
|
@go install github.com/editorconfig-checker/editorconfig-checker/\
|
||||||
|
cmd/editorconfig-checker@latest
|
||||||
@echo "All tools installed successfully!"
|
@echo "All tools installed successfully!"
|
||||||
|
|
||||||
# Run linters
|
# Run linters
|
||||||
@@ -40,6 +53,8 @@ lint:
|
|||||||
lint-fix:
|
lint-fix:
|
||||||
@echo "Running gofumpt..."
|
@echo "Running gofumpt..."
|
||||||
@gofumpt -l -w .
|
@gofumpt -l -w .
|
||||||
|
@echo "Running golines..."
|
||||||
|
@golines -w -m 120 --base-formatter="gofumpt" --shorten-comments .
|
||||||
@echo "Running goimports..."
|
@echo "Running goimports..."
|
||||||
@goimports -w -local github.com/ivuorinen/gibidify .
|
@goimports -w -local github.com/ivuorinen/gibidify .
|
||||||
@echo "Running go fmt..."
|
@echo "Running go fmt..."
|
||||||
@@ -47,32 +62,46 @@ lint-fix:
|
|||||||
@echo "Running go mod tidy..."
|
@echo "Running go mod tidy..."
|
||||||
@go mod tidy
|
@go mod tidy
|
||||||
@echo "Running shfmt formatting..."
|
@echo "Running shfmt formatting..."
|
||||||
@shfmt -w -i 2 -ci .
|
@shfmt -w -i 0 -ci .
|
||||||
@echo "Running golangci-lint with --fix..."
|
@echo "Running golangci-lint with --fix..."
|
||||||
@golangci-lint run --fix ./...
|
@golangci-lint run --fix ./...
|
||||||
@echo "Auto-fix completed. Running final lint check..."
|
@echo "Auto-fix completed. Running final lint check..."
|
||||||
@golangci-lint run ./...
|
@golangci-lint run ./...
|
||||||
|
@echo "Running revive..."
|
||||||
|
@revive -config revive.toml -formatter friendly ./...
|
||||||
@echo "Running checkmake..."
|
@echo "Running checkmake..."
|
||||||
@checkmake --config=.checkmake Makefile
|
@checkmake --config=.checkmake Makefile
|
||||||
@echo "Running yamllint..."
|
@echo "Running yamllint..."
|
||||||
@yamllint -c .yamllint .
|
@yamllint .
|
||||||
|
|
||||||
# Run linters with verbose output
|
# Run linters with verbose output
|
||||||
lint-verbose:
|
lint-verbose:
|
||||||
@echo "Running golangci-lint (verbose)..."
|
@echo "Running golangci-lint (verbose)..."
|
||||||
@golangci-lint run -v ./...
|
@golangci-lint run -v ./...
|
||||||
@echo "Running checkmake (verbose)..."
|
@echo "Running checkmake (verbose)..."
|
||||||
@checkmake --config=.checkmake --format="{{.Line}}:{{.Rule}}:{{.Violation}}" Makefile
|
@checkmake --config=.checkmake \
|
||||||
|
--format="{{.Line}}:{{.Rule}}:{{.Violation}}" Makefile
|
||||||
@echo "Running shfmt check (verbose)..."
|
@echo "Running shfmt check (verbose)..."
|
||||||
@shfmt -d .
|
@shfmt -d .
|
||||||
@echo "Running yamllint (verbose)..."
|
@echo "Running yamllint (verbose)..."
|
||||||
@yamllint -c .yamllint -f parsable .
|
@yamllint .
|
||||||
|
|
||||||
# Run tests
|
# Run tests
|
||||||
test:
|
test:
|
||||||
@echo "Running tests..."
|
@echo "Running tests..."
|
||||||
@go test -race -v ./...
|
@go test -race -v ./...
|
||||||
|
|
||||||
|
# Run tests with coverage output
|
||||||
|
test-coverage:
|
||||||
|
@echo "Running tests with coverage..."
|
||||||
|
@go test -race -v -coverprofile=coverage.out -covermode=atomic ./...
|
||||||
|
@echo ""
|
||||||
|
@echo "Coverage summary:"
|
||||||
|
@go tool cover -func=coverage.out | grep total:
|
||||||
|
@echo ""
|
||||||
|
@echo "Full coverage report saved to: coverage.out"
|
||||||
|
@echo "To view HTML report, run: make coverage"
|
||||||
|
|
||||||
# Run tests with coverage
|
# Run tests with coverage
|
||||||
coverage:
|
coverage:
|
||||||
@echo "Running tests with coverage..."
|
@echo "Running tests with coverage..."
|
||||||
@@ -94,8 +123,6 @@ clean:
|
|||||||
@echo "Clean complete"
|
@echo "Clean complete"
|
||||||
|
|
||||||
# CI-specific targets
|
# CI-specific targets
|
||||||
.PHONY: ci-lint ci-test
|
|
||||||
|
|
||||||
ci-lint:
|
ci-lint:
|
||||||
@golangci-lint run --out-format=github-actions ./...
|
@golangci-lint run --out-format=github-actions ./...
|
||||||
|
|
||||||
@@ -138,10 +165,34 @@ security:
|
|||||||
security-full:
|
security-full:
|
||||||
@echo "Running full security analysis..."
|
@echo "Running full security analysis..."
|
||||||
@./scripts/security-scan.sh
|
@./scripts/security-scan.sh
|
||||||
@echo "Running additional security checks..."
|
|
||||||
@golangci-lint run --enable-all --disable=depguard,exhaustruct,ireturn,varnamelen,wrapcheck --timeout=10m
|
|
||||||
|
|
||||||
vuln-check:
|
vuln-check:
|
||||||
@echo "Checking for dependency vulnerabilities..."
|
@echo "Checking for dependency vulnerabilities..."
|
||||||
@go install golang.org/x/vuln/cmd/govulncheck@latest
|
@go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
@govulncheck ./...
|
@govulncheck ./...
|
||||||
|
|
||||||
|
# Dependency management targets
|
||||||
|
deps-check:
|
||||||
|
@echo "Checking for available dependency updates..."
|
||||||
|
@echo ""
|
||||||
|
@echo "Direct dependencies:"
|
||||||
|
@go list -u -m all | grep -v "indirect" | column -t
|
||||||
|
@echo ""
|
||||||
|
@echo "Note: Run 'make deps-update' to update all dependencies"
|
||||||
|
|
||||||
|
deps-update:
|
||||||
|
@echo "Updating all dependencies to latest versions..."
|
||||||
|
@go get -u ./...
|
||||||
|
@go mod tidy
|
||||||
|
@echo ""
|
||||||
|
@echo "Dependencies updated successfully!"
|
||||||
|
@echo "Running tests to verify compatibility..."
|
||||||
|
@go test ./...
|
||||||
|
@echo ""
|
||||||
|
@echo "Update complete. Run 'make lint-fix && make test' to verify."
|
||||||
|
|
||||||
|
deps-tidy:
|
||||||
|
@echo "Cleaning up dependencies..."
|
||||||
|
@go mod tidy
|
||||||
|
@go mod verify
|
||||||
|
@echo "Dependencies cleaned and verified successfully!"
|
||||||
|
|||||||
18
README.md
18
README.md
@@ -32,15 +32,15 @@ go build -o gibidify .
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
./gibidify \
|
./gibidify \
|
||||||
-source <source_directory> \
|
-source <source_directory> \
|
||||||
-destination <output_file> \
|
-destination <output_file> \
|
||||||
-format markdown|json|yaml \
|
-format markdown|json|yaml \
|
||||||
-concurrency <num_workers> \
|
-concurrency <num_workers> \
|
||||||
--prefix="..." \
|
--prefix="..." \
|
||||||
--suffix="..." \
|
--suffix="..." \
|
||||||
--no-colors \
|
--no-colors \
|
||||||
--no-progress \
|
--no-progress \
|
||||||
--verbose
|
--verbose
|
||||||
```
|
```
|
||||||
|
|
||||||
Flags:
|
Flags:
|
||||||
|
|||||||
@@ -12,11 +12,11 @@ import (
|
|||||||
|
|
||||||
"github.com/ivuorinen/gibidify/config"
|
"github.com/ivuorinen/gibidify/config"
|
||||||
"github.com/ivuorinen/gibidify/fileproc"
|
"github.com/ivuorinen/gibidify/fileproc"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BenchmarkResult represents the results of a benchmark run.
|
// Result represents the results of a benchmark run.
|
||||||
type BenchmarkResult struct {
|
type Result struct {
|
||||||
Name string
|
Name string
|
||||||
Duration time.Duration
|
Duration time.Duration
|
||||||
FilesProcessed int
|
FilesProcessed int
|
||||||
@@ -42,14 +42,14 @@ type CPUStats struct {
|
|||||||
Goroutines int
|
Goroutines int
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkSuite represents a collection of benchmarks.
|
// Suite represents a collection of benchmarks.
|
||||||
type BenchmarkSuite struct {
|
type Suite struct {
|
||||||
Name string
|
Name string
|
||||||
Results []BenchmarkResult
|
Results []Result
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileCollectionBenchmark benchmarks file collection operations.
|
// FileCollectionBenchmark benchmarks file collection operations.
|
||||||
func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult, error) {
|
func FileCollectionBenchmark(sourceDir string, numFiles int) (*Result, error) {
|
||||||
// Load configuration to ensure proper file filtering
|
// Load configuration to ensure proper file filtering
|
||||||
config.LoadConfig()
|
config.LoadConfig()
|
||||||
|
|
||||||
@@ -58,7 +58,12 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
|
|||||||
if sourceDir == "" {
|
if sourceDir == "" {
|
||||||
tempDir, cleanupFunc, err := createBenchmarkFiles(numFiles)
|
tempDir, cleanupFunc, err := createBenchmarkFiles(numFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create benchmark files")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSAccess,
|
||||||
|
"failed to create benchmark files",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
cleanup = cleanupFunc
|
cleanup = cleanupFunc
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
@@ -74,7 +79,12 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
|
|||||||
// Run the file collection benchmark
|
// Run the file collection benchmark
|
||||||
files, err := fileproc.CollectFiles(sourceDir)
|
files, err := fileproc.CollectFiles(sourceDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "benchmark file collection failed")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"benchmark file collection failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
duration := time.Since(startTime)
|
duration := time.Since(startTime)
|
||||||
@@ -91,7 +101,7 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result := &BenchmarkResult{
|
result := &Result{
|
||||||
Name: "FileCollection",
|
Name: "FileCollection",
|
||||||
Duration: duration,
|
Duration: duration,
|
||||||
FilesProcessed: len(files),
|
FilesProcessed: len(files),
|
||||||
@@ -113,7 +123,9 @@ func FileCollectionBenchmark(sourceDir string, numFiles int) (*BenchmarkResult,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FileProcessingBenchmark benchmarks full file processing pipeline.
|
// FileProcessingBenchmark benchmarks full file processing pipeline.
|
||||||
func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (*BenchmarkResult, error) {
|
//
|
||||||
|
//revive:disable-next-line:function-length
|
||||||
|
func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (*Result, error) {
|
||||||
// Load configuration to ensure proper file filtering
|
// Load configuration to ensure proper file filtering
|
||||||
config.LoadConfig()
|
config.LoadConfig()
|
||||||
|
|
||||||
@@ -122,7 +134,12 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
|
|||||||
// Create temporary directory with test files
|
// Create temporary directory with test files
|
||||||
tempDir, cleanupFunc, err := createBenchmarkFiles(100)
|
tempDir, cleanupFunc, err := createBenchmarkFiles(100)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create benchmark files")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSAccess,
|
||||||
|
"failed to create benchmark files",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
cleanup = cleanupFunc
|
cleanup = cleanupFunc
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
@@ -132,7 +149,12 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
|
|||||||
// Create temporary output file
|
// Create temporary output file
|
||||||
outputFile, err := os.CreateTemp("", "benchmark_output_*."+format)
|
outputFile, err := os.CreateTemp("", "benchmark_output_*."+format)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileCreate, "failed to create benchmark output file")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileCreate,
|
||||||
|
"failed to create benchmark output file",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := outputFile.Close(); err != nil {
|
if err := outputFile.Close(); err != nil {
|
||||||
@@ -154,13 +176,29 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
|
|||||||
// Run the full processing pipeline
|
// Run the full processing pipeline
|
||||||
files, err := fileproc.CollectFiles(sourceDir)
|
files, err := fileproc.CollectFiles(sourceDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "benchmark file collection failed")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"benchmark file collection failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process files with concurrency
|
// Process files with concurrency
|
||||||
err = runProcessingPipeline(context.Background(), files, outputFile, format, concurrency, sourceDir)
|
err = runProcessingPipeline(context.Background(), processingConfig{
|
||||||
|
files: files,
|
||||||
|
outputFile: outputFile,
|
||||||
|
format: format,
|
||||||
|
concurrency: concurrency,
|
||||||
|
sourceDir: sourceDir,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "benchmark processing pipeline failed")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingFileRead,
|
||||||
|
"benchmark processing pipeline failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
duration := time.Since(startTime)
|
duration := time.Since(startTime)
|
||||||
@@ -177,7 +215,7 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result := &BenchmarkResult{
|
result := &Result{
|
||||||
Name: fmt.Sprintf("FileProcessing_%s_c%d", format, concurrency),
|
Name: fmt.Sprintf("FileProcessing_%s_c%d", format, concurrency),
|
||||||
Duration: duration,
|
Duration: duration,
|
||||||
FilesProcessed: len(files),
|
FilesProcessed: len(files),
|
||||||
@@ -199,16 +237,22 @@ func FileProcessingBenchmark(sourceDir string, format string, concurrency int) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ConcurrencyBenchmark benchmarks different concurrency levels.
|
// ConcurrencyBenchmark benchmarks different concurrency levels.
|
||||||
func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []int) (*BenchmarkSuite, error) {
|
func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []int) (*Suite, error) {
|
||||||
suite := &BenchmarkSuite{
|
suite := &Suite{
|
||||||
Name: "ConcurrencyBenchmark",
|
Name: "ConcurrencyBenchmark",
|
||||||
Results: make([]BenchmarkResult, 0, len(concurrencyLevels)),
|
Results: make([]Result, 0, len(concurrencyLevels)),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, concurrency := range concurrencyLevels {
|
for _, concurrency := range concurrencyLevels {
|
||||||
result, err := FileProcessingBenchmark(sourceDir, format, concurrency)
|
result, err := FileProcessingBenchmark(sourceDir, format, concurrency)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapErrorf(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed for level %d", concurrency)
|
return nil, gibidiutils.WrapErrorf(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"concurrency benchmark failed for level %d",
|
||||||
|
concurrency,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
suite.Results = append(suite.Results, *result)
|
suite.Results = append(suite.Results, *result)
|
||||||
}
|
}
|
||||||
@@ -217,16 +261,22 @@ func ConcurrencyBenchmark(sourceDir string, format string, concurrencyLevels []i
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FormatBenchmark benchmarks different output formats.
|
// FormatBenchmark benchmarks different output formats.
|
||||||
func FormatBenchmark(sourceDir string, formats []string) (*BenchmarkSuite, error) {
|
func FormatBenchmark(sourceDir string, formats []string) (*Suite, error) {
|
||||||
suite := &BenchmarkSuite{
|
suite := &Suite{
|
||||||
Name: "FormatBenchmark",
|
Name: "FormatBenchmark",
|
||||||
Results: make([]BenchmarkResult, 0, len(formats)),
|
Results: make([]Result, 0, len(formats)),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, format := range formats {
|
for _, format := range formats {
|
||||||
result, err := FileProcessingBenchmark(sourceDir, format, runtime.NumCPU())
|
result, err := FileProcessingBenchmark(sourceDir, format, runtime.NumCPU())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapErrorf(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed for format %s", format)
|
return nil, gibidiutils.WrapErrorf(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"format benchmark failed for format %s",
|
||||||
|
format,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
suite.Results = append(suite.Results, *result)
|
suite.Results = append(suite.Results, *result)
|
||||||
}
|
}
|
||||||
@@ -238,7 +288,12 @@ func FormatBenchmark(sourceDir string, formats []string) (*BenchmarkSuite, error
|
|||||||
func createBenchmarkFiles(numFiles int) (string, func(), error) {
|
func createBenchmarkFiles(numFiles int) (string, func(), error) {
|
||||||
tempDir, err := os.MkdirTemp("", "gibidify_benchmark_*")
|
tempDir, err := os.MkdirTemp("", "gibidify_benchmark_*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create temp directory")
|
return "", nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSAccess,
|
||||||
|
"failed to create temp directory",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup := func() {
|
cleanup := func() {
|
||||||
@@ -256,8 +311,15 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
|
|||||||
{".go", "package main\n\nfunc main() {\n\tprintln(\"Hello, World!\")\n}"},
|
{".go", "package main\n\nfunc main() {\n\tprintln(\"Hello, World!\")\n}"},
|
||||||
{".js", "console.log('Hello, World!');"},
|
{".js", "console.log('Hello, World!');"},
|
||||||
{".py", "print('Hello, World!')"},
|
{".py", "print('Hello, World!')"},
|
||||||
{".java", "public class Hello {\n\tpublic static void main(String[] args) {\n\t\tSystem.out.println(\"Hello, World!\");\n\t}\n}"},
|
{
|
||||||
{".cpp", "#include <iostream>\n\nint main() {\n\tstd::cout << \"Hello, World!\" << std::endl;\n\treturn 0;\n}"},
|
".java",
|
||||||
|
"public class Hello {\n\tpublic static void main(String[] args) {" +
|
||||||
|
"\n\t\tSystem.out.println(\"Hello, World!\");\n\t}\n}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
".cpp",
|
||||||
|
"#include <iostream>\n\nint main() {\n\tstd::cout << \"Hello, World!\" << std::endl;\n\treturn 0;\n}",
|
||||||
|
},
|
||||||
{".rs", "fn main() {\n\tprintln!(\"Hello, World!\");\n}"},
|
{".rs", "fn main() {\n\tprintln!(\"Hello, World!\");\n}"},
|
||||||
{".rb", "puts 'Hello, World!'"},
|
{".rb", "puts 'Hello, World!'"},
|
||||||
{".php", "<?php\necho 'Hello, World!';\n?>"},
|
{".php", "<?php\necho 'Hello, World!';\n?>"},
|
||||||
@@ -272,9 +334,14 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
|
|||||||
// Create subdirectories for some files
|
// Create subdirectories for some files
|
||||||
if i%10 == 0 {
|
if i%10 == 0 {
|
||||||
subdir := filepath.Join(tempDir, fmt.Sprintf("subdir_%d", i/10))
|
subdir := filepath.Join(tempDir, fmt.Sprintf("subdir_%d", i/10))
|
||||||
if err := os.MkdirAll(subdir, 0o755); err != nil {
|
if err := os.MkdirAll(subdir, 0o750); err != nil {
|
||||||
cleanup()
|
cleanup()
|
||||||
return "", nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to create subdirectory")
|
return "", nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSAccess,
|
||||||
|
"failed to create subdirectory",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
filename = filepath.Join(subdir, filename)
|
filename = filepath.Join(subdir, filename)
|
||||||
} else {
|
} else {
|
||||||
@@ -287,9 +354,14 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
|
|||||||
content += fmt.Sprintf("// Line %d\n%s\n", j, fileType.content)
|
content += fmt.Sprintf("// Line %d\n%s\n", j, fileType.content)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.WriteFile(filename, []byte(content), 0o644); err != nil {
|
if err := os.WriteFile(filename, []byte(content), 0o600); err != nil {
|
||||||
cleanup()
|
cleanup()
|
||||||
return "", nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileWrite, "failed to write benchmark file")
|
return "", nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileWrite,
|
||||||
|
"failed to write benchmark file",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,23 +369,41 @@ func createBenchmarkFiles(numFiles int) (string, func(), error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runProcessingPipeline runs the processing pipeline similar to main.go.
|
// runProcessingPipeline runs the processing pipeline similar to main.go.
|
||||||
func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.File, format string, concurrency int, sourceDir string) error {
|
// processingConfig holds configuration for processing pipeline.
|
||||||
fileCh := make(chan string, concurrency)
|
type processingConfig struct {
|
||||||
writeCh := make(chan fileproc.WriteRequest, concurrency)
|
files []string
|
||||||
|
outputFile *os.File
|
||||||
|
format string
|
||||||
|
concurrency int
|
||||||
|
sourceDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func runProcessingPipeline(ctx context.Context, config processingConfig) error {
|
||||||
|
fileCh := make(chan string, config.concurrency)
|
||||||
|
writeCh := make(chan fileproc.WriteRequest, config.concurrency)
|
||||||
writerDone := make(chan struct{})
|
writerDone := make(chan struct{})
|
||||||
|
|
||||||
// Start writer
|
// Start writer
|
||||||
go fileproc.StartWriter(outputFile, writeCh, writerDone, format, "", "")
|
go fileproc.StartWriter(config.outputFile, writeCh, writerDone, fileproc.WriterConfig{
|
||||||
|
Format: config.format,
|
||||||
|
Prefix: "",
|
||||||
|
Suffix: "",
|
||||||
|
})
|
||||||
|
|
||||||
// Get absolute path once
|
// Get absolute path once
|
||||||
absRoot, err := utils.GetAbsolutePath(sourceDir)
|
absRoot, err := gibidiutils.GetAbsolutePath(config.sourceDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSPathResolution, "failed to get absolute path for source directory")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSPathResolution,
|
||||||
|
"failed to get absolute path for source directory",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start workers with proper synchronization
|
// Start workers with proper synchronization
|
||||||
var workersDone sync.WaitGroup
|
var workersDone sync.WaitGroup
|
||||||
for i := 0; i < concurrency; i++ {
|
for i := 0; i < config.concurrency; i++ {
|
||||||
workersDone.Add(1)
|
workersDone.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer workersDone.Done()
|
defer workersDone.Done()
|
||||||
@@ -324,7 +414,7 @@ func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.F
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Send files to workers
|
// Send files to workers
|
||||||
for _, file := range files {
|
for _, file := range config.files {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
close(fileCh)
|
close(fileCh)
|
||||||
@@ -347,8 +437,8 @@ func runProcessingPipeline(ctx context.Context, files []string, outputFile *os.F
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintBenchmarkResult prints a formatted benchmark result.
|
// PrintResult prints a formatted benchmark result.
|
||||||
func PrintBenchmarkResult(result *BenchmarkResult) {
|
func PrintResult(result *Result) {
|
||||||
fmt.Printf("=== %s ===\n", result.Name)
|
fmt.Printf("=== %s ===\n", result.Name)
|
||||||
fmt.Printf("Duration: %v\n", result.Duration)
|
fmt.Printf("Duration: %v\n", result.Duration)
|
||||||
fmt.Printf("Files Processed: %d\n", result.FilesProcessed)
|
fmt.Printf("Files Processed: %d\n", result.FilesProcessed)
|
||||||
@@ -356,16 +446,17 @@ func PrintBenchmarkResult(result *BenchmarkResult) {
|
|||||||
fmt.Printf("Files/sec: %.2f\n", result.FilesPerSecond)
|
fmt.Printf("Files/sec: %.2f\n", result.FilesPerSecond)
|
||||||
fmt.Printf("Bytes/sec: %.2f MB/sec\n", result.BytesPerSecond/1024/1024)
|
fmt.Printf("Bytes/sec: %.2f MB/sec\n", result.BytesPerSecond/1024/1024)
|
||||||
fmt.Printf("Memory Usage: +%.2f MB (Sys: +%.2f MB)\n", result.MemoryUsage.AllocMB, result.MemoryUsage.SysMB)
|
fmt.Printf("Memory Usage: +%.2f MB (Sys: +%.2f MB)\n", result.MemoryUsage.AllocMB, result.MemoryUsage.SysMB)
|
||||||
fmt.Printf("GC Runs: %d (Pause: %v)\n", result.MemoryUsage.NumGC, time.Duration(result.MemoryUsage.PauseTotalNs))
|
pauseDuration := time.Duration(gibidiutils.SafeUint64ToInt64WithDefault(result.MemoryUsage.PauseTotalNs, 0))
|
||||||
|
fmt.Printf("GC Runs: %d (Pause: %v)\n", result.MemoryUsage.NumGC, pauseDuration)
|
||||||
fmt.Printf("Goroutines: %d\n", result.CPUUsage.Goroutines)
|
fmt.Printf("Goroutines: %d\n", result.CPUUsage.Goroutines)
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintBenchmarkSuite prints all results in a benchmark suite.
|
// PrintSuite prints all results in a benchmark suite.
|
||||||
func PrintBenchmarkSuite(suite *BenchmarkSuite) {
|
func PrintSuite(suite *Suite) {
|
||||||
fmt.Printf("=== %s ===\n", suite.Name)
|
fmt.Printf("=== %s ===\n", suite.Name)
|
||||||
for _, result := range suite.Results {
|
for i := range suite.Results {
|
||||||
PrintBenchmarkResult(&result)
|
PrintResult(&suite.Results[i])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -380,26 +471,41 @@ func RunAllBenchmarks(sourceDir string) error {
|
|||||||
fmt.Println("Running file collection benchmark...")
|
fmt.Println("Running file collection benchmark...")
|
||||||
result, err := FileCollectionBenchmark(sourceDir, 1000)
|
result, err := FileCollectionBenchmark(sourceDir, 1000)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file collection benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"file collection benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
PrintBenchmarkResult(result)
|
PrintResult(result)
|
||||||
|
|
||||||
// Format benchmarks
|
// Format benchmarks
|
||||||
fmt.Println("Running format benchmarks...")
|
fmt.Println("Running format benchmarks...")
|
||||||
formatSuite, err := FormatBenchmark(sourceDir, []string{"json", "yaml", "markdown"})
|
formatSuite, err := FormatBenchmark(sourceDir, []string{"json", "yaml", "markdown"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"format benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
PrintBenchmarkSuite(formatSuite)
|
PrintSuite(formatSuite)
|
||||||
|
|
||||||
// Concurrency benchmarks
|
// Concurrency benchmarks
|
||||||
fmt.Println("Running concurrency benchmarks...")
|
fmt.Println("Running concurrency benchmarks...")
|
||||||
concurrencyLevels := []int{1, 2, 4, 8, runtime.NumCPU()}
|
concurrencyLevels := []int{1, 2, 4, 8, runtime.NumCPU()}
|
||||||
concurrencySuite, err := ConcurrencyBenchmark(sourceDir, "json", concurrencyLevels)
|
concurrencySuite, err := ConcurrencyBenchmark(sourceDir, "json", concurrencyLevels)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"concurrency benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
PrintBenchmarkSuite(concurrencySuite)
|
PrintSuite(concurrencySuite)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
154
cli/errors.go
154
cli/errors.go
@@ -1,3 +1,4 @@
|
|||||||
|
// Package cli provides command-line interface utilities for gibidify.
|
||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -6,7 +7,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrorFormatter handles CLI-friendly error formatting with suggestions.
|
// ErrorFormatter handles CLI-friendly error formatting with suggestions.
|
||||||
@@ -19,6 +20,11 @@ func NewErrorFormatter(ui *UIManager) *ErrorFormatter {
|
|||||||
return &ErrorFormatter{ui: ui}
|
return &ErrorFormatter{ui: ui}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Suggestion messages for error formatting.
|
||||||
|
const (
|
||||||
|
suggestionCheckPermissions = " %s Check file/directory permissions\n"
|
||||||
|
)
|
||||||
|
|
||||||
// FormatError formats an error with context and suggestions.
|
// FormatError formats an error with context and suggestions.
|
||||||
func (ef *ErrorFormatter) FormatError(err error) {
|
func (ef *ErrorFormatter) FormatError(err error) {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -26,7 +32,8 @@ func (ef *ErrorFormatter) FormatError(err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Handle structured errors
|
// Handle structured errors
|
||||||
if structErr, ok := err.(*utils.StructuredError); ok {
|
var structErr *gibidiutils.StructuredError
|
||||||
|
if errors.As(err, &structErr) {
|
||||||
ef.formatStructuredError(structErr)
|
ef.formatStructuredError(structErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -36,12 +43,12 @@ func (ef *ErrorFormatter) FormatError(err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// formatStructuredError formats a structured error with context and suggestions.
|
// formatStructuredError formats a structured error with context and suggestions.
|
||||||
func (ef *ErrorFormatter) formatStructuredError(err *utils.StructuredError) {
|
func (ef *ErrorFormatter) formatStructuredError(err *gibidiutils.StructuredError) {
|
||||||
// Print main error
|
// Print main error
|
||||||
ef.ui.PrintError("Error: %s", err.Message)
|
ef.ui.PrintError("Error: %s", err.Message)
|
||||||
|
|
||||||
// Print error type and code
|
// Print error type and code
|
||||||
if err.Type != utils.ErrorTypeUnknown || err.Code != "" {
|
if err.Type != gibidiutils.ErrorTypeUnknown || err.Code != "" {
|
||||||
ef.ui.PrintInfo("Type: %s, Code: %s", err.Type.String(), err.Code)
|
ef.ui.PrintInfo("Type: %s, Code: %s", err.Type.String(), err.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,15 +76,15 @@ func (ef *ErrorFormatter) formatGenericError(err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// provideSuggestions provides helpful suggestions based on the error.
|
// provideSuggestions provides helpful suggestions based on the error.
|
||||||
func (ef *ErrorFormatter) provideSuggestions(err *utils.StructuredError) {
|
func (ef *ErrorFormatter) provideSuggestions(err *gibidiutils.StructuredError) {
|
||||||
switch err.Type {
|
switch err.Type {
|
||||||
case utils.ErrorTypeFileSystem:
|
case gibidiutils.ErrorTypeFileSystem:
|
||||||
ef.provideFileSystemSuggestions(err)
|
ef.provideFileSystemSuggestions(err)
|
||||||
case utils.ErrorTypeValidation:
|
case gibidiutils.ErrorTypeValidation:
|
||||||
ef.provideValidationSuggestions(err)
|
ef.provideValidationSuggestions(err)
|
||||||
case utils.ErrorTypeProcessing:
|
case gibidiutils.ErrorTypeProcessing:
|
||||||
ef.provideProcessingSuggestions(err)
|
ef.provideProcessingSuggestions(err)
|
||||||
case utils.ErrorTypeIO:
|
case gibidiutils.ErrorTypeIO:
|
||||||
ef.provideIOSuggestions(err)
|
ef.provideIOSuggestions(err)
|
||||||
default:
|
default:
|
||||||
ef.provideDefaultSuggestions()
|
ef.provideDefaultSuggestions()
|
||||||
@@ -85,17 +92,17 @@ func (ef *ErrorFormatter) provideSuggestions(err *utils.StructuredError) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// provideFileSystemSuggestions provides suggestions for file system errors.
|
// provideFileSystemSuggestions provides suggestions for file system errors.
|
||||||
func (ef *ErrorFormatter) provideFileSystemSuggestions(err *utils.StructuredError) {
|
func (ef *ErrorFormatter) provideFileSystemSuggestions(err *gibidiutils.StructuredError) {
|
||||||
filePath := err.FilePath
|
filePath := err.FilePath
|
||||||
|
|
||||||
ef.ui.PrintWarning("Suggestions:")
|
ef.ui.PrintWarning("Suggestions:")
|
||||||
|
|
||||||
switch err.Code {
|
switch err.Code {
|
||||||
case utils.CodeFSAccess:
|
case gibidiutils.CodeFSAccess:
|
||||||
ef.suggestFileAccess(filePath)
|
ef.suggestFileAccess(filePath)
|
||||||
case utils.CodeFSPathResolution:
|
case gibidiutils.CodeFSPathResolution:
|
||||||
ef.suggestPathResolution(filePath)
|
ef.suggestPathResolution(filePath)
|
||||||
case utils.CodeFSNotFound:
|
case gibidiutils.CodeFSNotFound:
|
||||||
ef.suggestFileNotFound(filePath)
|
ef.suggestFileNotFound(filePath)
|
||||||
default:
|
default:
|
||||||
ef.suggestFileSystemGeneral(filePath)
|
ef.suggestFileSystemGeneral(filePath)
|
||||||
@@ -103,91 +110,91 @@ func (ef *ErrorFormatter) provideFileSystemSuggestions(err *utils.StructuredErro
|
|||||||
}
|
}
|
||||||
|
|
||||||
// provideValidationSuggestions provides suggestions for validation errors.
|
// provideValidationSuggestions provides suggestions for validation errors.
|
||||||
func (ef *ErrorFormatter) provideValidationSuggestions(err *utils.StructuredError) {
|
func (ef *ErrorFormatter) provideValidationSuggestions(err *gibidiutils.StructuredError) {
|
||||||
ef.ui.PrintWarning("Suggestions:")
|
ef.ui.PrintWarning("Suggestions:")
|
||||||
|
|
||||||
switch err.Code {
|
switch err.Code {
|
||||||
case utils.CodeValidationFormat:
|
case gibidiutils.CodeValidationFormat:
|
||||||
ef.ui.printf(" • Use a supported format: markdown, json, yaml\n")
|
ef.ui.printf(" %s Use a supported format: markdown, json, yaml\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Example: -format markdown\n")
|
ef.ui.printf(" %s Example: -format markdown\n", gibidiutils.IconBullet)
|
||||||
case utils.CodeValidationSize:
|
case gibidiutils.CodeValidationSize:
|
||||||
ef.ui.printf(" • Increase file size limit in config.yaml\n")
|
ef.ui.printf(" %s Increase file size limit in config.yaml\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Use smaller files or exclude large files\n")
|
ef.ui.printf(" %s Use smaller files or exclude large files\n", gibidiutils.IconBullet)
|
||||||
default:
|
default:
|
||||||
ef.ui.printf(" • Check your command line arguments\n")
|
ef.ui.printf(" %s Check your command line arguments\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Run with --help for usage information\n")
|
ef.ui.printf(" %s Run with --help for usage information\n", gibidiutils.IconBullet)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// provideProcessingSuggestions provides suggestions for processing errors.
|
// provideProcessingSuggestions provides suggestions for processing errors.
|
||||||
func (ef *ErrorFormatter) provideProcessingSuggestions(err *utils.StructuredError) {
|
func (ef *ErrorFormatter) provideProcessingSuggestions(err *gibidiutils.StructuredError) {
|
||||||
ef.ui.PrintWarning("Suggestions:")
|
ef.ui.PrintWarning("Suggestions:")
|
||||||
|
|
||||||
switch err.Code {
|
switch err.Code {
|
||||||
case utils.CodeProcessingCollection:
|
case gibidiutils.CodeProcessingCollection:
|
||||||
ef.ui.printf(" • Check if the source directory exists and is readable\n")
|
ef.ui.printf(" %s Check if the source directory exists and is readable\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Verify directory permissions\n")
|
ef.ui.printf(" %s Verify directory permissions\n", gibidiutils.IconBullet)
|
||||||
case utils.CodeProcessingFileRead:
|
case gibidiutils.CodeProcessingFileRead:
|
||||||
ef.ui.printf(" • Check file permissions\n")
|
ef.ui.printf(" %s Check file permissions\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Verify the file is not corrupted\n")
|
ef.ui.printf(" %s Verify the file is not corrupted\n", gibidiutils.IconBullet)
|
||||||
default:
|
default:
|
||||||
ef.ui.printf(" • Try reducing concurrency: -concurrency 1\n")
|
ef.ui.printf(" %s Try reducing concurrency: -concurrency 1\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Check available system resources\n")
|
ef.ui.printf(" %s Check available system resources\n", gibidiutils.IconBullet)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// provideIOSuggestions provides suggestions for I/O errors.
|
// provideIOSuggestions provides suggestions for I/O errors.
|
||||||
func (ef *ErrorFormatter) provideIOSuggestions(err *utils.StructuredError) {
|
func (ef *ErrorFormatter) provideIOSuggestions(err *gibidiutils.StructuredError) {
|
||||||
ef.ui.PrintWarning("Suggestions:")
|
ef.ui.PrintWarning("Suggestions:")
|
||||||
|
|
||||||
switch err.Code {
|
switch err.Code {
|
||||||
case utils.CodeIOFileCreate:
|
case gibidiutils.CodeIOFileCreate:
|
||||||
ef.ui.printf(" • Check if the destination directory exists\n")
|
ef.ui.printf(" %s Check if the destination directory exists\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Verify write permissions for the output file\n")
|
ef.ui.printf(" %s Verify write permissions for the output file\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Ensure sufficient disk space\n")
|
ef.ui.printf(" %s Ensure sufficient disk space\n", gibidiutils.IconBullet)
|
||||||
case utils.CodeIOWrite:
|
case gibidiutils.CodeIOWrite:
|
||||||
ef.ui.printf(" • Check available disk space\n")
|
ef.ui.printf(" %s Check available disk space\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Verify write permissions\n")
|
ef.ui.printf(" %s Verify write permissions\n", gibidiutils.IconBullet)
|
||||||
default:
|
default:
|
||||||
ef.ui.printf(" • Check file/directory permissions\n")
|
ef.ui.printf(suggestionCheckPermissions, gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Verify available disk space\n")
|
ef.ui.printf(" %s Verify available disk space\n", gibidiutils.IconBullet)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper methods for specific suggestions
|
// Helper methods for specific suggestions
|
||||||
func (ef *ErrorFormatter) suggestFileAccess(filePath string) {
|
func (ef *ErrorFormatter) suggestFileAccess(filePath string) {
|
||||||
ef.ui.printf(" • Check if the path exists: %s\n", filePath)
|
ef.ui.printf(" %s Check if the path exists: %s\n", gibidiutils.IconBullet, filePath)
|
||||||
ef.ui.printf(" • Verify read permissions\n")
|
ef.ui.printf(" %s Verify read permissions\n", gibidiutils.IconBullet)
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
if stat, err := os.Stat(filePath); err == nil {
|
if stat, err := os.Stat(filePath); err == nil {
|
||||||
ef.ui.printf(" • Path exists but may not be accessible\n")
|
ef.ui.printf(" %s Path exists but may not be accessible\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Mode: %s\n", stat.Mode())
|
ef.ui.printf(" %s Mode: %s\n", gibidiutils.IconBullet, stat.Mode())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ef *ErrorFormatter) suggestPathResolution(filePath string) {
|
func (ef *ErrorFormatter) suggestPathResolution(filePath string) {
|
||||||
ef.ui.printf(" • Use an absolute path instead of relative\n")
|
ef.ui.printf(" %s Use an absolute path instead of relative\n", gibidiutils.IconBullet)
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
if abs, err := filepath.Abs(filePath); err == nil {
|
if abs, err := filepath.Abs(filePath); err == nil {
|
||||||
ef.ui.printf(" • Try: %s\n", abs)
|
ef.ui.printf(" %s Try: %s\n", gibidiutils.IconBullet, abs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ef *ErrorFormatter) suggestFileNotFound(filePath string) {
|
func (ef *ErrorFormatter) suggestFileNotFound(filePath string) {
|
||||||
ef.ui.printf(" • Check if the file/directory exists: %s\n", filePath)
|
ef.ui.printf(" %s Check if the file/directory exists: %s\n", gibidiutils.IconBullet, filePath)
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
dir := filepath.Dir(filePath)
|
dir := filepath.Dir(filePath)
|
||||||
if entries, err := os.ReadDir(dir); err == nil {
|
if entries, err := os.ReadDir(dir); err == nil {
|
||||||
ef.ui.printf(" • Similar files in %s:\n", dir)
|
ef.ui.printf(" %s Similar files in %s:\n", gibidiutils.IconBullet, dir)
|
||||||
count := 0
|
count := 0
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if count >= 3 {
|
if count >= 3 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if strings.Contains(entry.Name(), filepath.Base(filePath)) {
|
if strings.Contains(entry.Name(), filepath.Base(filePath)) {
|
||||||
ef.ui.printf(" - %s\n", entry.Name())
|
ef.ui.printf(" %s %s\n", gibidiutils.IconBullet, entry.Name())
|
||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,18 +203,18 @@ func (ef *ErrorFormatter) suggestFileNotFound(filePath string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (ef *ErrorFormatter) suggestFileSystemGeneral(filePath string) {
|
func (ef *ErrorFormatter) suggestFileSystemGeneral(filePath string) {
|
||||||
ef.ui.printf(" • Check file/directory permissions\n")
|
ef.ui.printf(suggestionCheckPermissions, gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Verify the path is correct\n")
|
ef.ui.printf(" %s Verify the path is correct\n", gibidiutils.IconBullet)
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
ef.ui.printf(" • Path: %s\n", filePath)
|
ef.ui.printf(" %s Path: %s\n", gibidiutils.IconBullet, filePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// provideDefaultSuggestions provides general suggestions.
|
// provideDefaultSuggestions provides general suggestions.
|
||||||
func (ef *ErrorFormatter) provideDefaultSuggestions() {
|
func (ef *ErrorFormatter) provideDefaultSuggestions() {
|
||||||
ef.ui.printf(" • Check your command line arguments\n")
|
ef.ui.printf(" %s Check your command line arguments\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Run with --help for usage information\n")
|
ef.ui.printf(" %s Run with --help for usage information\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Try with -concurrency 1 to reduce resource usage\n")
|
ef.ui.printf(" %s Try with -concurrency 1 to reduce resource usage\n", gibidiutils.IconBullet)
|
||||||
}
|
}
|
||||||
|
|
||||||
// provideGenericSuggestions provides suggestions for generic errors.
|
// provideGenericSuggestions provides suggestions for generic errors.
|
||||||
@@ -219,14 +226,14 @@ func (ef *ErrorFormatter) provideGenericSuggestions(err error) {
|
|||||||
// Pattern matching for common errors
|
// Pattern matching for common errors
|
||||||
switch {
|
switch {
|
||||||
case strings.Contains(errorMsg, "permission denied"):
|
case strings.Contains(errorMsg, "permission denied"):
|
||||||
ef.ui.printf(" • Check file/directory permissions\n")
|
ef.ui.printf(suggestionCheckPermissions, gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Try running with appropriate privileges\n")
|
ef.ui.printf(" %s Try running with appropriate privileges\n", gibidiutils.IconBullet)
|
||||||
case strings.Contains(errorMsg, "no such file or directory"):
|
case strings.Contains(errorMsg, "no such file or directory"):
|
||||||
ef.ui.printf(" • Verify the file/directory path is correct\n")
|
ef.ui.printf(" %s Verify the file/directory path is correct\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Check if the file exists\n")
|
ef.ui.printf(" %s Check if the file exists\n", gibidiutils.IconBullet)
|
||||||
case strings.Contains(errorMsg, "flag") && strings.Contains(errorMsg, "redefined"):
|
case strings.Contains(errorMsg, "flag") && strings.Contains(errorMsg, "redefined"):
|
||||||
ef.ui.printf(" • This is likely a test environment issue\n")
|
ef.ui.printf(" %s This is likely a test environment issue\n", gibidiutils.IconBullet)
|
||||||
ef.ui.printf(" • Try running the command directly instead of in tests\n")
|
ef.ui.printf(" %s Try running the command directly instead of in tests\n", gibidiutils.IconBullet)
|
||||||
default:
|
default:
|
||||||
ef.provideDefaultSuggestions()
|
ef.provideDefaultSuggestions()
|
||||||
}
|
}
|
||||||
@@ -234,16 +241,16 @@ func (ef *ErrorFormatter) provideGenericSuggestions(err error) {
|
|||||||
|
|
||||||
// CLI-specific error types
|
// CLI-specific error types
|
||||||
|
|
||||||
// CLIMissingSourceError represents a missing source directory error.
|
// MissingSourceError represents a missing source directory error.
|
||||||
type CLIMissingSourceError struct{}
|
type MissingSourceError struct{}
|
||||||
|
|
||||||
func (e CLIMissingSourceError) Error() string {
|
func (e MissingSourceError) Error() string {
|
||||||
return "source directory is required"
|
return "source directory is required"
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCLIMissingSourceError creates a new CLI missing source error with suggestions.
|
// NewMissingSourceError creates a new CLI missing source error with suggestions.
|
||||||
func NewCLIMissingSourceError() error {
|
func NewMissingSourceError() error {
|
||||||
return &CLIMissingSourceError{}
|
return &MissingSourceError{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsUserError checks if an error is a user input error that should be handled gracefully.
|
// IsUserError checks if an error is a user input error that should be handled gracefully.
|
||||||
@@ -253,16 +260,17 @@ func IsUserError(err error) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check for specific user error types
|
// Check for specific user error types
|
||||||
var cliErr *CLIMissingSourceError
|
var cliErr *MissingSourceError
|
||||||
if errors.As(err, &cliErr) {
|
if errors.As(err, &cliErr) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for structured errors that are user-facing
|
// Check for structured errors that are user-facing
|
||||||
if structErr, ok := err.(*utils.StructuredError); ok {
|
var structErr *gibidiutils.StructuredError
|
||||||
return structErr.Type == utils.ErrorTypeValidation ||
|
if errors.As(err, &structErr) {
|
||||||
structErr.Code == utils.CodeValidationFormat ||
|
return structErr.Type == gibidiutils.ErrorTypeValidation ||
|
||||||
structErr.Code == utils.CodeValidationSize
|
structErr.Code == gibidiutils.CodeValidationFormat ||
|
||||||
|
structErr.Code == gibidiutils.CodeValidationSize
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check error message patterns
|
// Check error message patterns
|
||||||
|
|||||||
963
cli/errors_test.go
Normal file
963
cli/errors_test.go
Normal file
@@ -0,0 +1,963 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/fatih/color"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewErrorFormatter(t *testing.T) {
|
||||||
|
ui := &UIManager{
|
||||||
|
output: &bytes.Buffer{},
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := NewErrorFormatter(ui)
|
||||||
|
|
||||||
|
assert.NotNil(t, ef)
|
||||||
|
assert.Equal(t, ui, ef.ui)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatError(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
expectedOutput []string
|
||||||
|
notExpected []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "nil error",
|
||||||
|
err: nil,
|
||||||
|
expectedOutput: []string{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "structured error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSNotFound,
|
||||||
|
testErrFileNotFound,
|
||||||
|
"/test/file.txt",
|
||||||
|
map[string]interface{}{"size": 1024},
|
||||||
|
),
|
||||||
|
expectedOutput: []string{
|
||||||
|
gibidiutils.IconError + testErrorSuffix,
|
||||||
|
"FileSystem",
|
||||||
|
testErrFileNotFound,
|
||||||
|
"/test/file.txt",
|
||||||
|
"NOT_FOUND",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "generic error",
|
||||||
|
err: errors.New("something went wrong"),
|
||||||
|
expectedOutput: []string{gibidiutils.IconError + testErrorSuffix, "something went wrong"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrapped structured error",
|
||||||
|
err: gibidiutils.WrapError(
|
||||||
|
errors.New("inner error"),
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationRequired,
|
||||||
|
"validation failed",
|
||||||
|
),
|
||||||
|
expectedOutput: []string{
|
||||||
|
gibidiutils.IconError + testErrorSuffix,
|
||||||
|
"validation failed",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = true
|
||||||
|
t.Cleanup(func() { color.NoColor = prev })
|
||||||
|
|
||||||
|
ef := NewErrorFormatter(ui)
|
||||||
|
ef.FormatError(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, expected := range tt.expectedOutput {
|
||||||
|
assert.Contains(t, output, expected)
|
||||||
|
}
|
||||||
|
for _, notExpected := range tt.notExpected {
|
||||||
|
assert.NotContains(t, output, notExpected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatStructuredError(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err *gibidiutils.StructuredError
|
||||||
|
expectedOutput []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "filesystem error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSPermission,
|
||||||
|
testErrPermissionDenied,
|
||||||
|
"/etc/shadow",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedOutput: []string{
|
||||||
|
"FileSystem",
|
||||||
|
testErrPermissionDenied,
|
||||||
|
"/etc/shadow",
|
||||||
|
"PERMISSION_DENIED",
|
||||||
|
testSuggestionsHeader,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
testErrInvalidFormat,
|
||||||
|
"",
|
||||||
|
map[string]interface{}{"format": "xml"},
|
||||||
|
),
|
||||||
|
expectedOutput: []string{
|
||||||
|
"Validation",
|
||||||
|
testErrInvalidFormat,
|
||||||
|
"FORMAT",
|
||||||
|
testSuggestionsHeader,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "processing error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingFileRead,
|
||||||
|
"failed to read file",
|
||||||
|
"large.bin",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedOutput: []string{
|
||||||
|
"Processing",
|
||||||
|
"failed to read file",
|
||||||
|
"large.bin",
|
||||||
|
"FILE_READ",
|
||||||
|
testSuggestionsHeader,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IO error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileWrite,
|
||||||
|
"disk full",
|
||||||
|
"/output/result.txt",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedOutput: []string{
|
||||||
|
"IO",
|
||||||
|
"disk full",
|
||||||
|
"/output/result.txt",
|
||||||
|
"FILE_WRITE",
|
||||||
|
testSuggestionsHeader,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = true
|
||||||
|
t.Cleanup(func() { color.NoColor = prev })
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.formatStructuredError(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, expected := range tt.expectedOutput {
|
||||||
|
assert.Contains(t, output, expected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatGenericError(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = true
|
||||||
|
t.Cleanup(func() { color.NoColor = prev })
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.formatGenericError(errors.New("generic error message"))
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, gibidiutils.IconError+testErrorSuffix)
|
||||||
|
assert.Contains(t, output, "generic error message")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProvideSuggestions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err *gibidiutils.StructuredError
|
||||||
|
expectedSugges []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "filesystem permission error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSPermission,
|
||||||
|
testErrPermissionDenied,
|
||||||
|
"/root/file",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestVerifyPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "filesystem not found error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSNotFound,
|
||||||
|
testErrFileNotFound,
|
||||||
|
"/missing/file",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Check if the file/directory exists: /missing/file",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation format error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"unsupported format",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestFormat,
|
||||||
|
testSuggestFormatEx,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation path error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"invalid path",
|
||||||
|
"../../etc",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "processing file read error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingFileRead,
|
||||||
|
"read error",
|
||||||
|
"corrupted.dat",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Check file permissions",
|
||||||
|
"Verify the file is not corrupted",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IO file write error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileWrite,
|
||||||
|
"write failed",
|
||||||
|
"/output.txt",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestDiskSpace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown error type",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeUnknown,
|
||||||
|
"UNKNOWN",
|
||||||
|
"unknown error",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = true
|
||||||
|
t.Cleanup(func() { color.NoColor = prev })
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.provideSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, suggestion := range tt.expectedSugges {
|
||||||
|
assert.Contains(t, output, suggestion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProvideFileSystemSuggestions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err *gibidiutils.StructuredError
|
||||||
|
expectedSugges []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testErrPermissionDenied,
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSPermission,
|
||||||
|
testErrPermissionDenied,
|
||||||
|
"/root/secret",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestVerifyPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path resolution error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSPathResolution,
|
||||||
|
"path error",
|
||||||
|
"../../../etc",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Use an absolute path instead of relative",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testErrFileNotFound,
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.CodeFSNotFound,
|
||||||
|
"not found",
|
||||||
|
"/missing.txt",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Check if the file/directory exists: /missing.txt",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default filesystem error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
"OTHER_FS_ERROR",
|
||||||
|
testErrOther,
|
||||||
|
"/some/path",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestVerifyPath,
|
||||||
|
"Path: /some/path",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.provideFileSystemSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, suggestion := range tt.expectedSugges {
|
||||||
|
assert.Contains(t, output, suggestion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProvideValidationSuggestions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err *gibidiutils.StructuredError
|
||||||
|
expectedSugges []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "format validation",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
testErrInvalidFormat,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestFormat,
|
||||||
|
testSuggestFormatEx,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path validation",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"invalid path",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "size validation",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationSize,
|
||||||
|
"size error",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Increase file size limit in config.yaml",
|
||||||
|
"Use smaller files or exclude large files",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required validation",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationRequired,
|
||||||
|
"required",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default validation",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
"OTHER_VALIDATION",
|
||||||
|
"other",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.provideValidationSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, suggestion := range tt.expectedSugges {
|
||||||
|
assert.Contains(t, output, suggestion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProvideProcessingSuggestions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err *gibidiutils.StructuredError
|
||||||
|
expectedSugges []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file read error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingFileRead,
|
||||||
|
"read error",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Check file permissions",
|
||||||
|
"Verify the file is not corrupted",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "collection error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"collection error",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Check if the source directory exists and is readable",
|
||||||
|
"Verify directory permissions",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testErrEncoding,
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingEncode,
|
||||||
|
testErrEncoding,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Try reducing concurrency: -concurrency 1",
|
||||||
|
"Check available system resources",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default processing",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
"OTHER",
|
||||||
|
testErrOther,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Try reducing concurrency: -concurrency 1",
|
||||||
|
"Check available system resources",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.provideProcessingSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, suggestion := range tt.expectedSugges {
|
||||||
|
assert.Contains(t, output, suggestion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProvideIOSuggestions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err *gibidiutils.StructuredError
|
||||||
|
expectedSugges []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file create error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileCreate,
|
||||||
|
"create error",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Check if the destination directory exists",
|
||||||
|
"Verify write permissions for the output file",
|
||||||
|
"Ensure sufficient disk space",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file write error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileWrite,
|
||||||
|
"write error",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestDiskSpace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testErrEncoding,
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOEncoding,
|
||||||
|
testErrEncoding,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestDiskSpace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default IO error",
|
||||||
|
err: gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
"OTHER",
|
||||||
|
testErrOther,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
testSuggestDiskSpace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.provideIOSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, suggestion := range tt.expectedSugges {
|
||||||
|
assert.Contains(t, output, suggestion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProvideGenericSuggestions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
expectedSugges []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "permission error",
|
||||||
|
err: errors.New("permission denied accessing file"),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckPerms,
|
||||||
|
"Try running with appropriate privileges",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "not found error",
|
||||||
|
err: errors.New("no such file or directory"),
|
||||||
|
expectedSugges: []string{
|
||||||
|
"Verify the file/directory path is correct",
|
||||||
|
"Check if the file exists",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "memory error",
|
||||||
|
err: errors.New("out of memory"),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
testSuggestReduceConcur,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "timeout error",
|
||||||
|
err: errors.New("operation timed out"),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
testSuggestReduceConcur,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "connection error",
|
||||||
|
err: errors.New("connection refused"),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
testSuggestReduceConcur,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default error",
|
||||||
|
err: errors.New("unknown error occurred"),
|
||||||
|
expectedSugges: []string{
|
||||||
|
testSuggestCheckArgs,
|
||||||
|
testSuggestHelp,
|
||||||
|
testSuggestReduceConcur,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
ef.provideGenericSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
for _, suggestion := range tt.expectedSugges {
|
||||||
|
assert.Contains(t, output, suggestion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMissingSourceError(t *testing.T) {
|
||||||
|
err := &MissingSourceError{}
|
||||||
|
|
||||||
|
assert.Equal(t, "source directory is required", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewMissingSourceErrorType(t *testing.T) {
|
||||||
|
err := NewMissingSourceError()
|
||||||
|
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, "source directory is required", err.Error())
|
||||||
|
|
||||||
|
var msErr *MissingSourceError
|
||||||
|
ok := errors.As(err, &msErr)
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.NotNil(t, msErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test error formatting with colors enabled
|
||||||
|
func TestFormatErrorWithColors(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: true,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = false
|
||||||
|
t.Cleanup(func() { color.NoColor = prev })
|
||||||
|
|
||||||
|
ef := NewErrorFormatter(ui)
|
||||||
|
err := gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
testErrInvalidFormat,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
ef.FormatError(err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
// When colors are enabled, some output may go directly to stdout
|
||||||
|
// Check for suggestions that are captured in the buffer
|
||||||
|
assert.Contains(t, output, testSuggestFormat)
|
||||||
|
assert.Contains(t, output, testSuggestFormatEx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test wrapped error handling
|
||||||
|
func TestFormatWrappedError(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ef := NewErrorFormatter(ui)
|
||||||
|
|
||||||
|
innerErr := errors.New("inner error")
|
||||||
|
wrappedErr := gibidiutils.WrapError(
|
||||||
|
innerErr,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingFileRead,
|
||||||
|
"wrapper message",
|
||||||
|
)
|
||||||
|
|
||||||
|
ef.FormatError(wrappedErr)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, "wrapper message")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test all suggestion paths get called
|
||||||
|
func TestSuggestionPathCoverage(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
|
||||||
|
// Test all error types
|
||||||
|
errorTypes := []gibidiutils.ErrorType{
|
||||||
|
gibidiutils.ErrorTypeFileSystem,
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.ErrorTypeConfiguration,
|
||||||
|
gibidiutils.ErrorTypeUnknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, errType := range errorTypes {
|
||||||
|
t.Run(errType.String(), func(t *testing.T) {
|
||||||
|
buf.Reset()
|
||||||
|
err := gibidiutils.NewStructuredError(
|
||||||
|
errType,
|
||||||
|
"TEST_CODE",
|
||||||
|
"test error",
|
||||||
|
"/test/path",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
ef.provideSuggestions(err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
// Should have some suggestion output
|
||||||
|
assert.NotEmpty(t, output)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test suggestion helper functions with various inputs
|
||||||
|
func TestSuggestHelpers(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
testFunc func(*ErrorFormatter)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "suggestFileAccess",
|
||||||
|
testFunc: func(ef *ErrorFormatter) {
|
||||||
|
ef.suggestFileAccess("/root/file")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "suggestPathResolution",
|
||||||
|
testFunc: func(ef *ErrorFormatter) {
|
||||||
|
ef.suggestPathResolution("../../../etc")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "suggestFileNotFound",
|
||||||
|
testFunc: func(ef *ErrorFormatter) {
|
||||||
|
ef.suggestFileNotFound("/missing")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "suggestFileSystemGeneral",
|
||||||
|
testFunc: func(ef *ErrorFormatter) {
|
||||||
|
ef.suggestFileSystemGeneral("/path")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "provideDefaultSuggestions",
|
||||||
|
testFunc: func(ef *ErrorFormatter) {
|
||||||
|
ef.provideDefaultSuggestions()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
|
||||||
|
tt.testFunc(ef)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
// Each should produce some output
|
||||||
|
assert.NotEmpty(t, output)
|
||||||
|
// Should contain bullet point
|
||||||
|
assert.Contains(t, output, gibidiutils.IconBullet)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test edge cases in error message analysis
|
||||||
|
func TestGenericSuggestionsEdgeCases(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{"empty message", errors.New("")},
|
||||||
|
{"very long message", errors.New(strings.Repeat("error ", 100))},
|
||||||
|
{"special characters", errors.New("error!@#$%^&*()")},
|
||||||
|
{"newlines", errors.New("error\nwith\nnewlines")},
|
||||||
|
{"unicode", errors.New("error with 中文 characters")},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
ef := &ErrorFormatter{ui: ui}
|
||||||
|
|
||||||
|
// Should not panic
|
||||||
|
ef.provideGenericSuggestions(tt.err)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
// Should have some output
|
||||||
|
assert.NotEmpty(t, output)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
28
cli/flags.go
28
cli/flags.go
@@ -5,7 +5,7 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/config"
|
"github.com/ivuorinen/gibidify/config"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Flags holds CLI flags values.
|
// Flags holds CLI flags values.
|
||||||
@@ -39,8 +39,10 @@ func ParseFlags() (*Flags, error) {
|
|||||||
flag.StringVar(&flags.Prefix, "prefix", "", "Text to add at the beginning of the output file")
|
flag.StringVar(&flags.Prefix, "prefix", "", "Text to add at the beginning of the output file")
|
||||||
flag.StringVar(&flags.Suffix, "suffix", "", "Text to add at the end of the output file")
|
flag.StringVar(&flags.Suffix, "suffix", "", "Text to add at the end of the output file")
|
||||||
flag.StringVar(&flags.Format, "format", "markdown", "Output format (json, markdown, yaml)")
|
flag.StringVar(&flags.Format, "format", "markdown", "Output format (json, markdown, yaml)")
|
||||||
flag.IntVar(&flags.Concurrency, "concurrency", runtime.NumCPU(),
|
flag.IntVar(
|
||||||
"Number of concurrent workers (default: number of CPU cores)")
|
&flags.Concurrency, "concurrency", runtime.NumCPU(),
|
||||||
|
"Number of concurrent workers (default: number of CPU cores)",
|
||||||
|
)
|
||||||
flag.BoolVar(&flags.NoColors, "no-colors", false, "Disable colored output")
|
flag.BoolVar(&flags.NoColors, "no-colors", false, "Disable colored output")
|
||||||
flag.BoolVar(&flags.NoProgress, "no-progress", false, "Disable progress bars")
|
flag.BoolVar(&flags.NoProgress, "no-progress", false, "Disable progress bars")
|
||||||
flag.BoolVar(&flags.Verbose, "verbose", false, "Enable verbose output")
|
flag.BoolVar(&flags.Verbose, "verbose", false, "Enable verbose output")
|
||||||
@@ -63,11 +65,11 @@ func ParseFlags() (*Flags, error) {
|
|||||||
// validate validates the CLI flags.
|
// validate validates the CLI flags.
|
||||||
func (f *Flags) validate() error {
|
func (f *Flags) validate() error {
|
||||||
if f.SourceDir == "" {
|
if f.SourceDir == "" {
|
||||||
return NewCLIMissingSourceError()
|
return NewMissingSourceError()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate source path for security
|
// Validate source path for security
|
||||||
if err := utils.ValidateSourcePath(f.SourceDir); err != nil {
|
if err := gibidiutils.ValidateSourcePath(f.SourceDir); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,28 +79,20 @@ func (f *Flags) validate() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Validate concurrency
|
// Validate concurrency
|
||||||
if err := config.ValidateConcurrency(f.Concurrency); err != nil {
|
return config.ValidateConcurrency(f.Concurrency)
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// setDefaultDestination sets the default destination if not provided.
|
// setDefaultDestination sets the default destination if not provided.
|
||||||
func (f *Flags) setDefaultDestination() error {
|
func (f *Flags) setDefaultDestination() error {
|
||||||
if f.Destination == "" {
|
if f.Destination == "" {
|
||||||
absRoot, err := utils.GetAbsolutePath(f.SourceDir)
|
absRoot, err := gibidiutils.GetAbsolutePath(f.SourceDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
baseName := utils.GetBaseName(absRoot)
|
baseName := gibidiutils.GetBaseName(absRoot)
|
||||||
f.Destination = baseName + "." + f.Format
|
f.Destination = baseName + "." + f.Format
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate destination path for security
|
// Validate destination path for security
|
||||||
if err := utils.ValidateDestinationPath(f.Destination); err != nil {
|
return gibidiutils.ValidateDestinationPath(f.Destination)
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
366
cli/flags_test.go
Normal file
366
cli/flags_test.go
Normal file
@@ -0,0 +1,366 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseFlags(t *testing.T) {
|
||||||
|
// Save original command line args and restore after test
|
||||||
|
oldArgs := os.Args
|
||||||
|
oldFlagsParsed := flagsParsed
|
||||||
|
defer func() {
|
||||||
|
os.Args = oldArgs
|
||||||
|
flagsParsed = oldFlagsParsed
|
||||||
|
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
|
||||||
|
}()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
expectedError string
|
||||||
|
validate func(t *testing.T, f *Flags)
|
||||||
|
setup func(t *testing.T)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid flags with all options",
|
||||||
|
args: []string{
|
||||||
|
"gibidify",
|
||||||
|
testFlagSource, "", // will set to tempDir in test body
|
||||||
|
"-destination", "output.md",
|
||||||
|
"-format", "json",
|
||||||
|
testFlagConcurrency, "4",
|
||||||
|
"-prefix", "prefix",
|
||||||
|
"-suffix", "suffix",
|
||||||
|
"-no-colors",
|
||||||
|
"-no-progress",
|
||||||
|
"-verbose",
|
||||||
|
},
|
||||||
|
validate: nil, // set in test body using closure
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing source directory",
|
||||||
|
args: []string{"gibidify"},
|
||||||
|
expectedError: testErrSourceRequired,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid format",
|
||||||
|
args: []string{
|
||||||
|
"gibidify",
|
||||||
|
testFlagSource, "", // will set to tempDir in test body
|
||||||
|
"-format", "invalid",
|
||||||
|
},
|
||||||
|
expectedError: "unsupported output format: invalid",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid concurrency (zero)",
|
||||||
|
args: []string{
|
||||||
|
"gibidify",
|
||||||
|
testFlagSource, "", // will set to tempDir in test body
|
||||||
|
testFlagConcurrency, "0",
|
||||||
|
},
|
||||||
|
expectedError: "concurrency (0) must be at least 1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid concurrency (too high)",
|
||||||
|
args: []string{
|
||||||
|
"gibidify",
|
||||||
|
testFlagSource, "", // will set to tempDir in test body
|
||||||
|
testFlagConcurrency, "200",
|
||||||
|
},
|
||||||
|
// Set maxConcurrency so the upper bound is enforced
|
||||||
|
expectedError: "concurrency (200) exceeds maximum (128)",
|
||||||
|
setup: func(t *testing.T) {
|
||||||
|
orig := viper.Get("maxConcurrency")
|
||||||
|
viper.Set("maxConcurrency", 128)
|
||||||
|
t.Cleanup(func() { viper.Set("maxConcurrency", orig) })
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path traversal in source",
|
||||||
|
args: []string{
|
||||||
|
"gibidify",
|
||||||
|
testFlagSource, testPathTraversalPath,
|
||||||
|
},
|
||||||
|
expectedError: testErrPathTraversal,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default values",
|
||||||
|
args: []string{
|
||||||
|
"gibidify",
|
||||||
|
testFlagSource, "", // will set to tempDir in test body
|
||||||
|
},
|
||||||
|
validate: nil, // set in test body using closure
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
// Reset flags for each test
|
||||||
|
flagsParsed = false
|
||||||
|
globalFlags = nil
|
||||||
|
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
|
||||||
|
|
||||||
|
// Create a local copy of args to avoid corrupting shared test data
|
||||||
|
args := append([]string{}, tt.args...)
|
||||||
|
|
||||||
|
// Use t.TempDir for source directory if needed
|
||||||
|
tempDir := ""
|
||||||
|
for i := range args {
|
||||||
|
if i > 0 && args[i-1] == testFlagSource && args[i] == "" {
|
||||||
|
tempDir = t.TempDir()
|
||||||
|
args[i] = tempDir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
os.Args = args
|
||||||
|
|
||||||
|
// Set validate closure if needed (for tempDir)
|
||||||
|
if tt.name == "valid flags with all options" {
|
||||||
|
tt.validate = func(t *testing.T, f *Flags) {
|
||||||
|
assert.Equal(t, tempDir, f.SourceDir)
|
||||||
|
assert.Equal(t, "output.md", f.Destination)
|
||||||
|
assert.Equal(t, "json", f.Format)
|
||||||
|
assert.Equal(t, 4, f.Concurrency)
|
||||||
|
assert.Equal(t, "prefix", f.Prefix)
|
||||||
|
assert.Equal(t, "suffix", f.Suffix)
|
||||||
|
assert.True(t, f.NoColors)
|
||||||
|
assert.True(t, f.NoProgress)
|
||||||
|
assert.True(t, f.Verbose)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tt.name == "default values" {
|
||||||
|
tt.validate = func(t *testing.T, f *Flags) {
|
||||||
|
assert.Equal(t, tempDir, f.SourceDir)
|
||||||
|
assert.Equal(t, "markdown", f.Format)
|
||||||
|
assert.Equal(t, runtime.NumCPU(), f.Concurrency)
|
||||||
|
assert.Equal(t, "", f.Prefix)
|
||||||
|
assert.Equal(t, "", f.Suffix)
|
||||||
|
assert.False(t, f.NoColors)
|
||||||
|
assert.False(t, f.NoProgress)
|
||||||
|
assert.False(t, f.Verbose)
|
||||||
|
// Destination should be set by setDefaultDestination
|
||||||
|
assert.NotEmpty(t, f.Destination)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call setup if present (e.g. for maxConcurrency)
|
||||||
|
if tt.setup != nil {
|
||||||
|
tt.setup(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
flags, err := ParseFlags()
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
if assert.Error(t, err) {
|
||||||
|
assert.Contains(t, err.Error(), tt.expectedError)
|
||||||
|
}
|
||||||
|
assert.Nil(t, flags)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.NotNil(t, flags)
|
||||||
|
if tt.validate != nil {
|
||||||
|
tt.validate(t, flags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFlagsValidate(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
flags *Flags
|
||||||
|
setupFunc func(t *testing.T, f *Flags)
|
||||||
|
expectedError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing source directory",
|
||||||
|
flags: &Flags{},
|
||||||
|
expectedError: testErrSourceRequired,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid format",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "invalid",
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
expectedError: "unsupported output format: invalid",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid concurrency",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "markdown",
|
||||||
|
Concurrency: 0,
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
expectedError: "concurrency (0) must be at least 1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path traversal attempt",
|
||||||
|
flags: &Flags{
|
||||||
|
SourceDir: testPathTraversalPath,
|
||||||
|
Format: "markdown",
|
||||||
|
},
|
||||||
|
expectedError: testErrPathTraversal,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid flags",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "json",
|
||||||
|
Concurrency: 4,
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if tt.setupFunc != nil {
|
||||||
|
tt.setupFunc(t, tt.flags)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := tt.flags.validate()
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), tt.expectedError)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetDefaultDestination(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
flags *Flags
|
||||||
|
setupFunc func(t *testing.T, f *Flags)
|
||||||
|
expectedDest string
|
||||||
|
expectedError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "default destination for directory",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "markdown",
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
expectedDest: "", // will check suffix below
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default destination for json format",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "json",
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
expectedDest: "", // will check suffix below
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "provided destination unchanged",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "markdown",
|
||||||
|
Destination: "custom-output.txt",
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
expectedDest: "custom-output.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path traversal in destination",
|
||||||
|
flags: &Flags{
|
||||||
|
Format: "markdown",
|
||||||
|
Destination: testPathTraversalPath,
|
||||||
|
},
|
||||||
|
setupFunc: func(t *testing.T, f *Flags) {
|
||||||
|
f.SourceDir = t.TempDir()
|
||||||
|
},
|
||||||
|
expectedError: testErrPathTraversal,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if tt.setupFunc != nil {
|
||||||
|
tt.setupFunc(t, tt.flags)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := tt.flags.setDefaultDestination()
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), tt.expectedError)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
switch {
|
||||||
|
case tt.expectedDest != "":
|
||||||
|
assert.Equal(t, tt.expectedDest, tt.flags.Destination)
|
||||||
|
case tt.flags.Format == "json":
|
||||||
|
assert.True(
|
||||||
|
t, strings.HasSuffix(tt.flags.Destination, ".json"),
|
||||||
|
"expected %q to have suffix .json", tt.flags.Destination,
|
||||||
|
)
|
||||||
|
case tt.flags.Format == "markdown":
|
||||||
|
assert.True(
|
||||||
|
t, strings.HasSuffix(tt.flags.Destination, ".markdown"),
|
||||||
|
"expected %q to have suffix .markdown", tt.flags.Destination,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFlagsSingleton(t *testing.T) {
|
||||||
|
// Save original state
|
||||||
|
oldFlagsParsed := flagsParsed
|
||||||
|
oldGlobalFlags := globalFlags
|
||||||
|
defer func() {
|
||||||
|
flagsParsed = oldFlagsParsed
|
||||||
|
globalFlags = oldGlobalFlags
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Test singleton behavior
|
||||||
|
flagsParsed = true
|
||||||
|
expectedFlags := &Flags{
|
||||||
|
SourceDir: "/test",
|
||||||
|
Format: "json",
|
||||||
|
Concurrency: 2,
|
||||||
|
}
|
||||||
|
globalFlags = expectedFlags
|
||||||
|
|
||||||
|
// Should return cached flags without parsing
|
||||||
|
flags, err := ParseFlags()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, expectedFlags, flags)
|
||||||
|
assert.Same(t, globalFlags, flags)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewMissingSourceError(t *testing.T) {
|
||||||
|
err := NewMissingSourceError()
|
||||||
|
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Equal(t, testErrSourceRequired, err.Error())
|
||||||
|
|
||||||
|
// Check if it's the right type
|
||||||
|
var missingSourceError *MissingSourceError
|
||||||
|
ok := errors.As(err, &missingSourceError)
|
||||||
|
assert.True(t, ok)
|
||||||
|
}
|
||||||
@@ -8,14 +8,19 @@ import (
|
|||||||
|
|
||||||
"github.com/ivuorinen/gibidify/config"
|
"github.com/ivuorinen/gibidify/config"
|
||||||
"github.com/ivuorinen/gibidify/fileproc"
|
"github.com/ivuorinen/gibidify/fileproc"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// collectFiles collects all files to be processed.
|
// collectFiles collects all files to be processed.
|
||||||
func (p *Processor) collectFiles() ([]string, error) {
|
func (p *Processor) collectFiles() ([]string, error) {
|
||||||
files, err := fileproc.CollectFiles(p.flags.SourceDir)
|
files, err := fileproc.CollectFiles(p.flags.SourceDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "error collecting files")
|
return nil, gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"error collecting files",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
logrus.Infof("Found %d files to process", len(files))
|
logrus.Infof("Found %d files to process", len(files))
|
||||||
return files, nil
|
return files, nil
|
||||||
@@ -30,9 +35,9 @@ func (p *Processor) validateFileCollection(files []string) error {
|
|||||||
// Check file count limit
|
// Check file count limit
|
||||||
maxFiles := config.GetMaxFiles()
|
maxFiles := config.GetMaxFiles()
|
||||||
if len(files) > maxFiles {
|
if len(files) > maxFiles {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitFiles,
|
gibidiutils.CodeResourceLimitFiles,
|
||||||
fmt.Sprintf("file count (%d) exceeds maximum limit (%d)", len(files), maxFiles),
|
fmt.Sprintf("file count (%d) exceeds maximum limit (%d)", len(files), maxFiles),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -51,10 +56,14 @@ func (p *Processor) validateFileCollection(files []string) error {
|
|||||||
if fileInfo, err := os.Stat(filePath); err == nil {
|
if fileInfo, err := os.Stat(filePath); err == nil {
|
||||||
totalSize += fileInfo.Size()
|
totalSize += fileInfo.Size()
|
||||||
if totalSize > maxTotalSize {
|
if totalSize > maxTotalSize {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitTotalSize,
|
gibidiutils.CodeResourceLimitTotalSize,
|
||||||
fmt.Sprintf("total file size (%d bytes) would exceed maximum limit (%d bytes)", totalSize, maxTotalSize),
|
fmt.Sprintf(
|
||||||
|
"total file size (%d bytes) would exceed maximum limit (%d bytes)",
|
||||||
|
totalSize,
|
||||||
|
maxTotalSize,
|
||||||
|
),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
"total_size": totalSize,
|
"total_size": totalSize,
|
||||||
@@ -74,4 +83,4 @@ func (p *Processor) validateFileCollection(files []string) error {
|
|||||||
|
|
||||||
logrus.Infof("Pre-validation passed: %d files, %d MB total", len(files), totalSize/1024/1024)
|
logrus.Infof("Pre-validation passed: %d files, %d MB total", len(files), totalSize/1024/1024)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/fileproc"
|
"github.com/ivuorinen/gibidify/fileproc"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Process executes the main file processing workflow.
|
// Process executes the main file processing workflow.
|
||||||
@@ -16,7 +16,9 @@ func (p *Processor) Process(ctx context.Context) error {
|
|||||||
defer overallCancel()
|
defer overallCancel()
|
||||||
|
|
||||||
// Configure file type registry
|
// Configure file type registry
|
||||||
p.configureFileTypes()
|
if err := p.configureFileTypes(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Print startup info with colors
|
// Print startup info with colors
|
||||||
p.ui.PrintHeader("🚀 Starting gibidify")
|
p.ui.PrintHeader("🚀 Starting gibidify")
|
||||||
@@ -55,7 +57,7 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
utils.LogError("Error closing output file", outFile.Close())
|
gibidiutils.LogError("Error closing output file", outFile.Close())
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Initialize back-pressure and channels
|
// Initialize back-pressure and channels
|
||||||
@@ -65,7 +67,11 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error {
|
|||||||
writerDone := make(chan struct{})
|
writerDone := make(chan struct{})
|
||||||
|
|
||||||
// Start writer
|
// Start writer
|
||||||
go fileproc.StartWriter(outFile, writeCh, writerDone, p.flags.Format, p.flags.Prefix, p.flags.Suffix)
|
go fileproc.StartWriter(outFile, writeCh, writerDone, fileproc.WriterConfig{
|
||||||
|
Format: p.flags.Format,
|
||||||
|
Prefix: p.flags.Prefix,
|
||||||
|
Suffix: p.flags.Suffix,
|
||||||
|
})
|
||||||
|
|
||||||
// Start workers
|
// Start workers
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -92,9 +98,13 @@ func (p *Processor) processFiles(ctx context.Context, files []string) error {
|
|||||||
// createOutputFile creates the output file.
|
// createOutputFile creates the output file.
|
||||||
func (p *Processor) createOutputFile() (*os.File, error) {
|
func (p *Processor) createOutputFile() (*os.File, error) {
|
||||||
// Destination path has been validated in CLI flags validation for path traversal attempts
|
// Destination path has been validated in CLI flags validation for path traversal attempts
|
||||||
outFile, err := os.Create(p.flags.Destination) // #nosec G304 - destination is validated in flags.validate()
|
// #nosec G304 - destination is validated in flags.validate()
|
||||||
|
outFile, err := os.Create(p.flags.Destination)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOFileCreate, "failed to create output file").WithFilePath(p.flags.Destination)
|
return nil, gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOFileCreate,
|
||||||
|
"failed to create output file",
|
||||||
|
).WithFilePath(p.flags.Destination)
|
||||||
}
|
}
|
||||||
return outFile, nil
|
return outFile, nil
|
||||||
}
|
}
|
||||||
|
|||||||
265
cli/processor_simple_test.go
Normal file
265
cli/processor_simple_test.go
Normal file
@@ -0,0 +1,265 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/fileproc"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProcessorSimple(t *testing.T) {
|
||||||
|
t.Run("NewProcessor", func(t *testing.T) {
|
||||||
|
flags := &Flags{
|
||||||
|
SourceDir: "/tmp/test",
|
||||||
|
Destination: "output.md",
|
||||||
|
Format: "markdown",
|
||||||
|
Concurrency: 2,
|
||||||
|
NoColors: true,
|
||||||
|
NoProgress: true,
|
||||||
|
Verbose: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
p := NewProcessor(flags)
|
||||||
|
|
||||||
|
assert.NotNil(t, p)
|
||||||
|
assert.Equal(t, flags, p.flags)
|
||||||
|
assert.NotNil(t, p.ui)
|
||||||
|
assert.NotNil(t, p.backpressure)
|
||||||
|
assert.NotNil(t, p.resourceMonitor)
|
||||||
|
assert.False(t, p.ui.enableColors)
|
||||||
|
assert.False(t, p.ui.enableProgress)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ConfigureFileTypes", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should not panic or error
|
||||||
|
err := p.configureFileTypes()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.NotNil(t, p)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("CreateOutputFile", func(t *testing.T) {
|
||||||
|
// Create temp file path
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
outputPath := filepath.Join(tempDir, "output.txt")
|
||||||
|
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{
|
||||||
|
Destination: outputPath,
|
||||||
|
},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := p.createOutputFile()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.NotNil(t, file)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
err = file.Close()
|
||||||
|
require.NoError(t, err)
|
||||||
|
err = os.Remove(outputPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ValidateFileCollection", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty collection should be valid (just checks limits)
|
||||||
|
err := p.validateFileCollection([]string{})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// Small collection should be valid
|
||||||
|
err = p.validateFileCollection([]string{
|
||||||
|
testFilePath1,
|
||||||
|
testFilePath2,
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("CollectFiles_EmptyDir", func(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{
|
||||||
|
SourceDir: tempDir,
|
||||||
|
},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := p.collectFiles()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Empty(t, files)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("CollectFiles_WithFiles", func(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
// Create test files
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(tempDir, "test1.go"), []byte("package main"), 0o600))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(tempDir, "test2.go"), []byte("package test"), 0o600))
|
||||||
|
|
||||||
|
// Set config so no files are ignored, and restore after test
|
||||||
|
origIgnoreDirs := viper.Get("ignoreDirectories")
|
||||||
|
origFileSizeLimit := viper.Get("fileSizeLimit")
|
||||||
|
viper.Set("ignoreDirectories", []string{})
|
||||||
|
viper.Set("fileSizeLimit", 1024*1024*10) // 10MB
|
||||||
|
t.Cleanup(func() {
|
||||||
|
viper.Set("ignoreDirectories", origIgnoreDirs)
|
||||||
|
viper.Set("fileSizeLimit", origFileSizeLimit)
|
||||||
|
})
|
||||||
|
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{
|
||||||
|
SourceDir: tempDir,
|
||||||
|
},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := p.collectFiles()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, files, 2)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("SendFiles", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
backpressure: fileproc.NewBackpressureManager(),
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
fileCh := make(chan string, 3)
|
||||||
|
files := []string{
|
||||||
|
testFilePath1,
|
||||||
|
testFilePath2,
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
// Send files in a goroutine since it might block
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
err := p.sendFiles(ctx, files, fileCh)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Read all files from channel
|
||||||
|
var received []string
|
||||||
|
for i := 0; i < len(files); i++ {
|
||||||
|
file := <-fileCh
|
||||||
|
received = append(received, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, len(files), len(received))
|
||||||
|
|
||||||
|
// Wait for sendFiles goroutine to finish (and close fileCh)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
// Now channel should be closed
|
||||||
|
_, ok := <-fileCh
|
||||||
|
assert.False(t, ok, "channel should be closed")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("WaitForCompletion", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
writeCh := make(chan fileproc.WriteRequest)
|
||||||
|
writerDone := make(chan struct{})
|
||||||
|
|
||||||
|
// Simulate writer finishing
|
||||||
|
go func() {
|
||||||
|
<-writeCh // Wait for close
|
||||||
|
close(writerDone)
|
||||||
|
}()
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
// Start and finish immediately
|
||||||
|
wg.Add(1)
|
||||||
|
wg.Done()
|
||||||
|
|
||||||
|
// Should complete without hanging
|
||||||
|
p.waitForCompletion(&wg, writeCh, writerDone)
|
||||||
|
assert.NotNil(t, p)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("LogFinalStats", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{
|
||||||
|
Verbose: true,
|
||||||
|
},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
resourceMonitor: fileproc.NewResourceMonitor(),
|
||||||
|
backpressure: fileproc.NewBackpressureManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should not panic
|
||||||
|
p.logFinalStats()
|
||||||
|
assert.NotNil(t, p)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test error handling scenarios
|
||||||
|
func TestProcessorErrors(t *testing.T) {
|
||||||
|
t.Run("CreateOutputFile_InvalidPath", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{
|
||||||
|
Destination: "/root/cannot-write-here.txt",
|
||||||
|
},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := p.createOutputFile()
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Nil(t, file)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("CollectFiles_NonExistentDir", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
flags: &Flags{
|
||||||
|
SourceDir: "/non/existent/path",
|
||||||
|
},
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := p.collectFiles()
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Nil(t, files)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("SendFiles_WithCancellation", func(t *testing.T) {
|
||||||
|
p := &Processor{
|
||||||
|
backpressure: fileproc.NewBackpressureManager(),
|
||||||
|
ui: NewUIManager(),
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
fileCh := make(chan string) // Unbuffered to force blocking
|
||||||
|
|
||||||
|
files := []string{
|
||||||
|
testFilePath1,
|
||||||
|
testFilePath2,
|
||||||
|
"/test/file3.go",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cancel immediately
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
err := p.sendFiles(ctx, files, fileCh)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Equal(t, context.Canceled, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -11,8 +11,12 @@ func (p *Processor) logFinalStats() {
|
|||||||
// Log back-pressure stats
|
// Log back-pressure stats
|
||||||
backpressureStats := p.backpressure.GetStats()
|
backpressureStats := p.backpressure.GetStats()
|
||||||
if backpressureStats.Enabled {
|
if backpressureStats.Enabled {
|
||||||
logrus.Infof("Back-pressure stats: processed=%d files, memory=%dMB/%dMB",
|
logrus.Infof(
|
||||||
backpressureStats.FilesProcessed, backpressureStats.CurrentMemoryUsage/1024/1024, backpressureStats.MaxMemoryUsage/1024/1024)
|
"Back-pressure stats: processed=%d files, memory=%dMB/%dMB",
|
||||||
|
backpressureStats.FilesProcessed,
|
||||||
|
backpressureStats.CurrentMemoryUsage/1024/1024,
|
||||||
|
backpressureStats.MaxMemoryUsage/1024/1024,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log resource monitoring stats
|
// Log resource monitoring stats
|
||||||
@@ -37,4 +41,4 @@ func (p *Processor) logFinalStats() {
|
|||||||
|
|
||||||
// Clean up resource monitor
|
// Clean up resource monitor
|
||||||
p.resourceMonitor.Close()
|
p.resourceMonitor.Close()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,15 +30,18 @@ func NewProcessor(flags *Flags) *Processor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// configureFileTypes configures the file type registry.
|
// configureFileTypes configures the file type registry.
|
||||||
func (p *Processor) configureFileTypes() {
|
func (p *Processor) configureFileTypes() error {
|
||||||
if config.GetFileTypesEnabled() {
|
if config.GetFileTypesEnabled() {
|
||||||
fileproc.ConfigureFromSettings(
|
if err := fileproc.ConfigureFromSettings(fileproc.RegistryConfig{
|
||||||
config.GetCustomImageExtensions(),
|
CustomImages: config.GetCustomImageExtensions(),
|
||||||
config.GetCustomBinaryExtensions(),
|
CustomBinary: config.GetCustomBinaryExtensions(),
|
||||||
config.GetCustomLanguages(),
|
CustomLanguages: config.GetCustomLanguages(),
|
||||||
config.GetDisabledImageExtensions(),
|
DisabledImages: config.GetDisabledImageExtensions(),
|
||||||
config.GetDisabledBinaryExtensions(),
|
DisabledBinary: config.GetDisabledBinaryExtensions(),
|
||||||
config.GetDisabledLanguageExtensions(),
|
DisabledLanguages: config.GetDisabledLanguageExtensions(),
|
||||||
)
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,11 +7,16 @@ import (
|
|||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/fileproc"
|
"github.com/ivuorinen/gibidify/fileproc"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// startWorkers starts the worker goroutines.
|
// startWorkers starts the worker goroutines.
|
||||||
func (p *Processor) startWorkers(ctx context.Context, wg *sync.WaitGroup, fileCh chan string, writeCh chan fileproc.WriteRequest) {
|
func (p *Processor) startWorkers(
|
||||||
|
ctx context.Context,
|
||||||
|
wg *sync.WaitGroup,
|
||||||
|
fileCh chan string,
|
||||||
|
writeCh chan fileproc.WriteRequest,
|
||||||
|
) {
|
||||||
for range p.flags.Concurrency {
|
for range p.flags.Concurrency {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go p.worker(ctx, wg, fileCh, writeCh)
|
go p.worker(ctx, wg, fileCh, writeCh)
|
||||||
@@ -19,7 +24,12 @@ func (p *Processor) startWorkers(ctx context.Context, wg *sync.WaitGroup, fileCh
|
|||||||
}
|
}
|
||||||
|
|
||||||
// worker is the worker goroutine function.
|
// worker is the worker goroutine function.
|
||||||
func (p *Processor) worker(ctx context.Context, wg *sync.WaitGroup, fileCh chan string, writeCh chan fileproc.WriteRequest) {
|
func (p *Processor) worker(
|
||||||
|
ctx context.Context,
|
||||||
|
wg *sync.WaitGroup,
|
||||||
|
fileCh chan string,
|
||||||
|
writeCh chan fileproc.WriteRequest,
|
||||||
|
) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
@@ -42,9 +52,9 @@ func (p *Processor) processFile(ctx context.Context, filePath string, writeCh ch
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
absRoot, err := utils.GetAbsolutePath(p.flags.SourceDir)
|
absRoot, err := gibidiutils.GetAbsolutePath(p.flags.SourceDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.LogError("Failed to get absolute path", err)
|
gibidiutils.LogError("Failed to get absolute path", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,8 +88,12 @@ func (p *Processor) sendFiles(ctx context.Context, files []string, fileCh chan s
|
|||||||
}
|
}
|
||||||
|
|
||||||
// waitForCompletion waits for all workers to complete.
|
// waitForCompletion waits for all workers to complete.
|
||||||
func (p *Processor) waitForCompletion(wg *sync.WaitGroup, writeCh chan fileproc.WriteRequest, writerDone chan struct{}) {
|
func (p *Processor) waitForCompletion(
|
||||||
|
wg *sync.WaitGroup,
|
||||||
|
writeCh chan fileproc.WriteRequest,
|
||||||
|
writerDone chan struct{},
|
||||||
|
) {
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
close(writeCh)
|
close(writeCh)
|
||||||
<-writerDone
|
<-writerDone
|
||||||
}
|
}
|
||||||
|
|||||||
68
cli/terminal_test_helpers.go
Normal file
68
cli/terminal_test_helpers.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// terminalEnvSetup defines environment variables for terminal detection tests.
|
||||||
|
type terminalEnvSetup struct {
|
||||||
|
Term string
|
||||||
|
CI string
|
||||||
|
GitHubActions string
|
||||||
|
NoColor string
|
||||||
|
ForceColor string
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply sets up the environment variables using t.Setenv.
|
||||||
|
func (e terminalEnvSetup) apply(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
// Always set all environment variables to ensure isolation
|
||||||
|
// Empty string explicitly unsets the variable in the test environment
|
||||||
|
t.Setenv("TERM", e.Term)
|
||||||
|
t.Setenv("CI", e.CI)
|
||||||
|
t.Setenv("GITHUB_ACTIONS", e.GitHubActions)
|
||||||
|
t.Setenv("NO_COLOR", e.NoColor)
|
||||||
|
t.Setenv("FORCE_COLOR", e.ForceColor)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Common terminal environment setups for reuse across tests.
|
||||||
|
var (
|
||||||
|
envDefaultTerminal = terminalEnvSetup{
|
||||||
|
Term: "xterm-256color",
|
||||||
|
CI: "",
|
||||||
|
NoColor: "",
|
||||||
|
ForceColor: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
envDumbTerminal = terminalEnvSetup{
|
||||||
|
Term: "dumb",
|
||||||
|
}
|
||||||
|
|
||||||
|
envCIWithoutGitHub = terminalEnvSetup{
|
||||||
|
Term: "xterm",
|
||||||
|
CI: "true",
|
||||||
|
GitHubActions: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
envGitHubActions = terminalEnvSetup{
|
||||||
|
Term: "xterm",
|
||||||
|
CI: "true",
|
||||||
|
GitHubActions: "true",
|
||||||
|
NoColor: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
envNoColor = terminalEnvSetup{
|
||||||
|
Term: "xterm-256color",
|
||||||
|
CI: "",
|
||||||
|
NoColor: "1",
|
||||||
|
ForceColor: "",
|
||||||
|
}
|
||||||
|
|
||||||
|
envForceColor = terminalEnvSetup{
|
||||||
|
Term: "dumb",
|
||||||
|
ForceColor: "1",
|
||||||
|
}
|
||||||
|
|
||||||
|
envEmptyTerm = terminalEnvSetup{
|
||||||
|
Term: "",
|
||||||
|
}
|
||||||
|
)
|
||||||
42
cli/test_constants.go
Normal file
42
cli/test_constants.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
// Test constants to avoid duplication in test files.
|
||||||
|
// These constants are used across multiple test files in the cli package.
|
||||||
|
const (
|
||||||
|
// Error messages
|
||||||
|
testErrFileNotFound = "file not found"
|
||||||
|
testErrPermissionDenied = "permission denied"
|
||||||
|
testErrInvalidFormat = "invalid format"
|
||||||
|
testErrOther = "other error"
|
||||||
|
testErrEncoding = "encoding error"
|
||||||
|
testErrSourceRequired = "source directory is required"
|
||||||
|
testErrPathTraversal = "path traversal attempt detected"
|
||||||
|
testPathTraversalPath = "../../../etc/passwd"
|
||||||
|
|
||||||
|
// Suggestion messages
|
||||||
|
testSuggestionsHeader = "Suggestions:"
|
||||||
|
testSuggestCheckPerms = "Check file/directory permissions"
|
||||||
|
testSuggestVerifyPath = "Verify the path is correct"
|
||||||
|
testSuggestFormat = "Use a supported format: markdown, json, yaml"
|
||||||
|
testSuggestFormatEx = "Example: -format markdown"
|
||||||
|
testSuggestCheckArgs = "Check your command line arguments"
|
||||||
|
testSuggestHelp = "Run with --help for usage information"
|
||||||
|
testSuggestDiskSpace = "Verify available disk space"
|
||||||
|
testSuggestReduceConcur = "Try with -concurrency 1 to reduce resource usage"
|
||||||
|
|
||||||
|
// UI test strings
|
||||||
|
testWithColors = "with colors"
|
||||||
|
testWithoutColors = "without colors"
|
||||||
|
testProcessingMsg = "Processing files"
|
||||||
|
|
||||||
|
// Flag names
|
||||||
|
testFlagSource = "-source"
|
||||||
|
testFlagConcurrency = "-concurrency"
|
||||||
|
|
||||||
|
// Test file paths
|
||||||
|
testFilePath1 = "/test/file1.go"
|
||||||
|
testFilePath2 = "/test/file2.go"
|
||||||
|
|
||||||
|
// Output markers
|
||||||
|
testErrorSuffix = " Error"
|
||||||
|
)
|
||||||
100
cli/ui.go
100
cli/ui.go
@@ -8,6 +8,8 @@ import (
|
|||||||
|
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// UIManager handles CLI user interface elements.
|
// UIManager handles CLI user interface elements.
|
||||||
@@ -44,23 +46,40 @@ func (ui *UIManager) StartProgress(total int, description string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ui.progressBar = progressbar.NewOptions(total,
|
// Set progress bar theme based on color support
|
||||||
progressbar.OptionSetWriter(ui.output),
|
var theme progressbar.Theme
|
||||||
progressbar.OptionSetDescription(description),
|
if ui.enableColors {
|
||||||
progressbar.OptionSetTheme(progressbar.Theme{
|
theme = progressbar.Theme{
|
||||||
Saucer: color.GreenString("█"),
|
Saucer: color.GreenString("█"),
|
||||||
SaucerHead: color.GreenString("█"),
|
SaucerHead: color.GreenString("█"),
|
||||||
SaucerPadding: " ",
|
SaucerPadding: " ",
|
||||||
BarStart: "[",
|
BarStart: "[",
|
||||||
BarEnd: "]",
|
BarEnd: "]",
|
||||||
}),
|
}
|
||||||
|
} else {
|
||||||
|
theme = progressbar.Theme{
|
||||||
|
Saucer: "█",
|
||||||
|
SaucerHead: "█",
|
||||||
|
SaucerPadding: " ",
|
||||||
|
BarStart: "[",
|
||||||
|
BarEnd: "]",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ui.progressBar = progressbar.NewOptions(
|
||||||
|
total,
|
||||||
|
progressbar.OptionSetWriter(ui.output),
|
||||||
|
progressbar.OptionSetDescription(description),
|
||||||
|
progressbar.OptionSetTheme(theme),
|
||||||
progressbar.OptionShowCount(),
|
progressbar.OptionShowCount(),
|
||||||
progressbar.OptionShowIts(),
|
progressbar.OptionShowIts(),
|
||||||
progressbar.OptionSetWidth(40),
|
progressbar.OptionSetWidth(40),
|
||||||
progressbar.OptionThrottle(100*time.Millisecond),
|
progressbar.OptionThrottle(100*time.Millisecond),
|
||||||
progressbar.OptionOnCompletion(func() {
|
progressbar.OptionOnCompletion(
|
||||||
_, _ = fmt.Fprint(ui.output, "\n")
|
func() {
|
||||||
}),
|
_, _ = fmt.Fprint(ui.output, "\n")
|
||||||
|
},
|
||||||
|
),
|
||||||
progressbar.OptionSetRenderBlankState(true),
|
progressbar.OptionSetRenderBlankState(true),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -80,40 +99,44 @@ func (ui *UIManager) FinishProgress() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintSuccess prints a success message in green.
|
// writeMessage writes a formatted message with optional colorization.
|
||||||
|
// It handles color enablement, formatting, writing to output, and error logging.
|
||||||
|
func (ui *UIManager) writeMessage(
|
||||||
|
icon, methodName, format string,
|
||||||
|
colorFunc func(string, ...interface{}) string,
|
||||||
|
args ...interface{},
|
||||||
|
) {
|
||||||
|
msg := icon + " " + format
|
||||||
|
var output string
|
||||||
|
if ui.enableColors && colorFunc != nil {
|
||||||
|
output = colorFunc(msg, args...)
|
||||||
|
} else {
|
||||||
|
output = fmt.Sprintf(msg, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := fmt.Fprintf(ui.output, "%s\n", output); err != nil {
|
||||||
|
gibidiutils.LogError(fmt.Sprintf("UIManager.%s: failed to write to output", methodName), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrintSuccess prints a success message in green (to ui.output if set).
|
||||||
func (ui *UIManager) PrintSuccess(format string, args ...interface{}) {
|
func (ui *UIManager) PrintSuccess(format string, args ...interface{}) {
|
||||||
if ui.enableColors {
|
ui.writeMessage(gibidiutils.IconSuccess, "PrintSuccess", format, color.GreenString, args...)
|
||||||
color.Green("✓ "+format, args...)
|
|
||||||
} else {
|
|
||||||
ui.printf("✓ "+format+"\n", args...)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintError prints an error message in red.
|
// PrintError prints an error message in red (to ui.output if set).
|
||||||
func (ui *UIManager) PrintError(format string, args ...interface{}) {
|
func (ui *UIManager) PrintError(format string, args ...interface{}) {
|
||||||
if ui.enableColors {
|
ui.writeMessage(gibidiutils.IconError, "PrintError", format, color.RedString, args...)
|
||||||
color.Red("✗ "+format, args...)
|
|
||||||
} else {
|
|
||||||
ui.printf("✗ "+format+"\n", args...)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintWarning prints a warning message in yellow.
|
// PrintWarning prints a warning message in yellow (to ui.output if set).
|
||||||
func (ui *UIManager) PrintWarning(format string, args ...interface{}) {
|
func (ui *UIManager) PrintWarning(format string, args ...interface{}) {
|
||||||
if ui.enableColors {
|
ui.writeMessage(gibidiutils.IconWarning, "PrintWarning", format, color.YellowString, args...)
|
||||||
color.Yellow("⚠ "+format, args...)
|
|
||||||
} else {
|
|
||||||
ui.printf("⚠ "+format+"\n", args...)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintInfo prints an info message in blue.
|
// PrintInfo prints an info message in blue (to ui.output if set).
|
||||||
func (ui *UIManager) PrintInfo(format string, args ...interface{}) {
|
func (ui *UIManager) PrintInfo(format string, args ...interface{}) {
|
||||||
if ui.enableColors {
|
ui.writeMessage(gibidiutils.IconInfo, "PrintInfo", format, color.BlueString, args...)
|
||||||
color.Blue("ℹ "+format, args...)
|
|
||||||
} else {
|
|
||||||
ui.printf("ℹ "+format+"\n", args...)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintHeader prints a header message in bold.
|
// PrintHeader prints a header message in bold.
|
||||||
@@ -127,6 +150,11 @@ func (ui *UIManager) PrintHeader(format string, args ...interface{}) {
|
|||||||
|
|
||||||
// isColorTerminal checks if the terminal supports colors.
|
// isColorTerminal checks if the terminal supports colors.
|
||||||
func isColorTerminal() bool {
|
func isColorTerminal() bool {
|
||||||
|
// Check if FORCE_COLOR is set
|
||||||
|
if os.Getenv("FORCE_COLOR") != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// Check common environment variables
|
// Check common environment variables
|
||||||
term := os.Getenv("TERM")
|
term := os.Getenv("TERM")
|
||||||
if term == "" || term == "dumb" {
|
if term == "" || term == "dumb" {
|
||||||
@@ -148,13 +176,7 @@ func isColorTerminal() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if FORCE_COLOR is set
|
return true
|
||||||
if os.Getenv("FORCE_COLOR") != "" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default to true for interactive terminals
|
|
||||||
return isInteractiveTerminal()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isInteractiveTerminal checks if we're running in an interactive terminal.
|
// isInteractiveTerminal checks if we're running in an interactive terminal.
|
||||||
|
|||||||
109
cli/ui_manager_test.go
Normal file
109
cli/ui_manager_test.go
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/fatih/color"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewUIManager(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
env terminalEnvSetup
|
||||||
|
expectedColors bool
|
||||||
|
expectedProgress bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "default terminal",
|
||||||
|
env: envDefaultTerminal,
|
||||||
|
expectedColors: true,
|
||||||
|
expectedProgress: false, // Not a tty in test environment
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dumb terminal",
|
||||||
|
env: envDumbTerminal,
|
||||||
|
expectedColors: false,
|
||||||
|
expectedProgress: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "CI environment without GitHub Actions",
|
||||||
|
env: envCIWithoutGitHub,
|
||||||
|
expectedColors: false,
|
||||||
|
expectedProgress: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GitHub Actions CI",
|
||||||
|
env: envGitHubActions,
|
||||||
|
expectedColors: true,
|
||||||
|
expectedProgress: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NO_COLOR set",
|
||||||
|
env: envNoColor,
|
||||||
|
expectedColors: false,
|
||||||
|
expectedProgress: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "FORCE_COLOR set",
|
||||||
|
env: envForceColor,
|
||||||
|
expectedColors: true,
|
||||||
|
expectedProgress: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
tt.env.apply(t)
|
||||||
|
|
||||||
|
ui := NewUIManager()
|
||||||
|
assert.NotNil(t, ui)
|
||||||
|
assert.NotNil(t, ui.output)
|
||||||
|
assert.Equal(t, tt.expectedColors, ui.enableColors, "color state mismatch")
|
||||||
|
assert.Equal(t, tt.expectedProgress, ui.enableProgress, "progress state mismatch")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetColorOutput(t *testing.T) {
|
||||||
|
// Capture original color.NoColor state and restore after test
|
||||||
|
orig := color.NoColor
|
||||||
|
defer func() { color.NoColor = orig }()
|
||||||
|
|
||||||
|
ui := &UIManager{output: os.Stderr}
|
||||||
|
|
||||||
|
// Test enabling colors
|
||||||
|
ui.SetColorOutput(true)
|
||||||
|
assert.False(t, color.NoColor)
|
||||||
|
assert.True(t, ui.enableColors)
|
||||||
|
|
||||||
|
// Test disabling colors
|
||||||
|
ui.SetColorOutput(false)
|
||||||
|
assert.True(t, color.NoColor)
|
||||||
|
assert.False(t, ui.enableColors)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetProgressOutput(t *testing.T) {
|
||||||
|
ui := &UIManager{output: os.Stderr}
|
||||||
|
|
||||||
|
// Test enabling progress
|
||||||
|
ui.SetProgressOutput(true)
|
||||||
|
assert.True(t, ui.enableProgress)
|
||||||
|
|
||||||
|
// Test disabling progress
|
||||||
|
ui.SetProgressOutput(false)
|
||||||
|
assert.False(t, ui.enableProgress)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintf(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ui.printf("Test %s %d", "output", 123)
|
||||||
|
|
||||||
|
assert.Equal(t, "Test output 123", buf.String())
|
||||||
|
}
|
||||||
245
cli/ui_print_test.go
Normal file
245
cli/ui_print_test.go
Normal file
@@ -0,0 +1,245 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/fatih/color"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPrintSuccess(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
enableColors bool
|
||||||
|
format string
|
||||||
|
args []interface{}
|
||||||
|
expectSymbol string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testWithColors,
|
||||||
|
enableColors: true,
|
||||||
|
format: "Operation %s",
|
||||||
|
args: []interface{}{"completed"},
|
||||||
|
expectSymbol: gibidiutils.IconSuccess,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testWithoutColors,
|
||||||
|
enableColors: false,
|
||||||
|
format: "Operation %s",
|
||||||
|
args: []interface{}{"completed"},
|
||||||
|
expectSymbol: gibidiutils.IconSuccess,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no arguments",
|
||||||
|
enableColors: true,
|
||||||
|
format: "Success",
|
||||||
|
args: nil,
|
||||||
|
expectSymbol: gibidiutils.IconSuccess,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: tt.enableColors,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = !tt.enableColors
|
||||||
|
defer func() { color.NoColor = prev }()
|
||||||
|
|
||||||
|
ui.PrintSuccess(tt.format, tt.args...)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, tt.expectSymbol)
|
||||||
|
if len(tt.args) > 0 {
|
||||||
|
assert.Contains(t, output, "completed")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintError(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
enableColors bool
|
||||||
|
format string
|
||||||
|
args []interface{}
|
||||||
|
expectSymbol string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testWithColors,
|
||||||
|
enableColors: true,
|
||||||
|
format: "Failed to %s",
|
||||||
|
args: []interface{}{"process"},
|
||||||
|
expectSymbol: gibidiutils.IconError,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testWithoutColors,
|
||||||
|
enableColors: false,
|
||||||
|
format: "Failed to %s",
|
||||||
|
args: []interface{}{"process"},
|
||||||
|
expectSymbol: gibidiutils.IconError,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: tt.enableColors,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
prev := color.NoColor
|
||||||
|
color.NoColor = !tt.enableColors
|
||||||
|
defer func() { color.NoColor = prev }()
|
||||||
|
|
||||||
|
ui.PrintError(tt.format, tt.args...)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, tt.expectSymbol)
|
||||||
|
if len(tt.args) > 0 {
|
||||||
|
assert.Contains(t, output, "process")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintWarning(t *testing.T) {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: true,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
ui.PrintWarning("This is a %s", "warning")
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, gibidiutils.IconWarning)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintInfo(t *testing.T) {
|
||||||
|
// Capture original color.NoColor state and restore after test
|
||||||
|
orig := color.NoColor
|
||||||
|
defer func() { color.NoColor = orig }()
|
||||||
|
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: true,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
color.NoColor = false
|
||||||
|
|
||||||
|
ui.PrintInfo("Information: %d items", 42)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, gibidiutils.IconInfo)
|
||||||
|
assert.Contains(t, output, "42")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintHeader(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
enableColors bool
|
||||||
|
format string
|
||||||
|
args []interface{}
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testWithColors,
|
||||||
|
enableColors: true,
|
||||||
|
format: "Header %s",
|
||||||
|
args: []interface{}{"Title"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testWithoutColors,
|
||||||
|
enableColors: false,
|
||||||
|
format: "Header %s",
|
||||||
|
args: []interface{}{"Title"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(t *testing.T) {
|
||||||
|
// Capture original color.NoColor state and restore after test
|
||||||
|
orig := color.NoColor
|
||||||
|
defer func() { color.NoColor = orig }()
|
||||||
|
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: tt.enableColors,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
color.NoColor = !tt.enableColors
|
||||||
|
|
||||||
|
ui.PrintHeader(tt.format, tt.args...)
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.Contains(t, output, "Title")
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test that all print methods handle newlines correctly
|
||||||
|
func TestPrintMethodsNewlines(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
method func(*UIManager, string, ...interface{})
|
||||||
|
symbol string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "PrintSuccess",
|
||||||
|
method: (*UIManager).PrintSuccess,
|
||||||
|
symbol: gibidiutils.IconSuccess,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PrintError",
|
||||||
|
method: (*UIManager).PrintError,
|
||||||
|
symbol: gibidiutils.IconError,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PrintWarning",
|
||||||
|
method: (*UIManager).PrintWarning,
|
||||||
|
symbol: gibidiutils.IconWarning,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PrintInfo",
|
||||||
|
method: (*UIManager).PrintInfo,
|
||||||
|
symbol: gibidiutils.IconInfo,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(t *testing.T) {
|
||||||
|
// Disable colors for consistent testing
|
||||||
|
oldNoColor := color.NoColor
|
||||||
|
color.NoColor = true
|
||||||
|
defer func() { color.NoColor = oldNoColor }()
|
||||||
|
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
ui := &UIManager{
|
||||||
|
enableColors: false,
|
||||||
|
output: buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
tt.method(ui, "Test message")
|
||||||
|
|
||||||
|
output := buf.String()
|
||||||
|
assert.True(t, strings.HasSuffix(output, "\n"))
|
||||||
|
assert.Contains(t, output, tt.symbol)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
147
cli/ui_progress_test.go
Normal file
147
cli/ui_progress_test.go
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStartProgress(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
total int
|
||||||
|
description string
|
||||||
|
enabled bool
|
||||||
|
expectBar bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "progress enabled with valid total",
|
||||||
|
total: 100,
|
||||||
|
description: testProcessingMsg,
|
||||||
|
enabled: true,
|
||||||
|
expectBar: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "progress disabled",
|
||||||
|
total: 100,
|
||||||
|
description: testProcessingMsg,
|
||||||
|
enabled: false,
|
||||||
|
expectBar: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero total",
|
||||||
|
total: 0,
|
||||||
|
description: testProcessingMsg,
|
||||||
|
enabled: true,
|
||||||
|
expectBar: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative total",
|
||||||
|
total: -5,
|
||||||
|
description: testProcessingMsg,
|
||||||
|
enabled: true,
|
||||||
|
expectBar: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(t *testing.T) {
|
||||||
|
ui := &UIManager{
|
||||||
|
enableProgress: tt.enabled,
|
||||||
|
output: &bytes.Buffer{},
|
||||||
|
}
|
||||||
|
|
||||||
|
ui.StartProgress(tt.total, tt.description)
|
||||||
|
|
||||||
|
if tt.expectBar {
|
||||||
|
assert.NotNil(t, ui.progressBar)
|
||||||
|
} else {
|
||||||
|
assert.Nil(t, ui.progressBar)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateProgress(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
setupBar bool
|
||||||
|
enabledProg bool
|
||||||
|
expectUpdate bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "with progress bar",
|
||||||
|
setupBar: true,
|
||||||
|
enabledProg: true,
|
||||||
|
expectUpdate: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "without progress bar",
|
||||||
|
setupBar: false,
|
||||||
|
enabledProg: false,
|
||||||
|
expectUpdate: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(_ *testing.T) {
|
||||||
|
ui := &UIManager{
|
||||||
|
enableProgress: tt.enabledProg,
|
||||||
|
output: &bytes.Buffer{},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.setupBar {
|
||||||
|
ui.StartProgress(10, "Test")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should not panic
|
||||||
|
ui.UpdateProgress(1)
|
||||||
|
|
||||||
|
// Multiple updates should not panic
|
||||||
|
ui.UpdateProgress(2)
|
||||||
|
ui.UpdateProgress(3)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFinishProgress(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
setupBar bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "with progress bar",
|
||||||
|
setupBar: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "without progress bar",
|
||||||
|
setupBar: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name, func(t *testing.T) {
|
||||||
|
ui := &UIManager{
|
||||||
|
enableProgress: true,
|
||||||
|
output: &bytes.Buffer{},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.setupBar {
|
||||||
|
ui.StartProgress(10, "Test")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should not panic
|
||||||
|
ui.FinishProgress()
|
||||||
|
|
||||||
|
// Bar should be cleared
|
||||||
|
assert.Nil(t, ui.progressBar)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
62
cli/ui_terminal_test.go
Normal file
62
cli/ui_terminal_test.go
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsColorTerminal(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
env terminalEnvSetup
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "dumb terminal",
|
||||||
|
env: envDumbTerminal,
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty TERM",
|
||||||
|
env: envEmptyTerm,
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "CI without GitHub Actions",
|
||||||
|
env: envCIWithoutGitHub,
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GitHub Actions",
|
||||||
|
env: envGitHubActions,
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NO_COLOR set",
|
||||||
|
env: envNoColor,
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "FORCE_COLOR set",
|
||||||
|
env: envForceColor,
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
tt.env.apply(t)
|
||||||
|
|
||||||
|
result := isColorTerminal()
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsInteractiveTerminal(t *testing.T) {
|
||||||
|
// This function checks if stderr is a terminal
|
||||||
|
// In test environment, it will typically return false
|
||||||
|
result := isInteractiveTerminal()
|
||||||
|
assert.False(t, result)
|
||||||
|
}
|
||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/benchmark"
|
"github.com/ivuorinen/gibidify/benchmark"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -26,7 +26,7 @@ func main() {
|
|||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
if err := runBenchmarks(); err != nil {
|
if err := runBenchmarks(); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "Benchmark failed: %v\n", err)
|
_, _ = fmt.Fprintf(os.Stderr, "Benchmark failed: %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -50,7 +50,10 @@ func runBenchmarks() error {
|
|||||||
case "format":
|
case "format":
|
||||||
return runFormatBenchmark()
|
return runFormatBenchmark()
|
||||||
default:
|
default:
|
||||||
return utils.NewValidationError(utils.CodeValidationFormat, "invalid benchmark type: "+*benchmarkType)
|
return gibidiutils.NewValidationError(
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"invalid benchmark type: "+*benchmarkType,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,9 +61,14 @@ func runCollectionBenchmark() error {
|
|||||||
fmt.Println("Running file collection benchmark...")
|
fmt.Println("Running file collection benchmark...")
|
||||||
result, err := benchmark.FileCollectionBenchmark(*sourceDir, *numFiles)
|
result, err := benchmark.FileCollectionBenchmark(*sourceDir, *numFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file collection benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"file collection benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
benchmark.PrintBenchmarkResult(result)
|
benchmark.PrintResult(result)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,24 +76,39 @@ func runProcessingBenchmark() error {
|
|||||||
fmt.Printf("Running file processing benchmark (format: %s, concurrency: %d)...\n", *format, *concurrency)
|
fmt.Printf("Running file processing benchmark (format: %s, concurrency: %d)...\n", *format, *concurrency)
|
||||||
result, err := benchmark.FileProcessingBenchmark(*sourceDir, *format, *concurrency)
|
result, err := benchmark.FileProcessingBenchmark(*sourceDir, *format, *concurrency)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "file processing benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"file processing benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
benchmark.PrintBenchmarkResult(result)
|
benchmark.PrintResult(result)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runConcurrencyBenchmark() error {
|
func runConcurrencyBenchmark() error {
|
||||||
concurrencyLevels, err := parseConcurrencyList(*concurrencyList)
|
concurrencyLevels, err := parseConcurrencyList(*concurrencyList)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeValidation, utils.CodeValidationFormat, "invalid concurrency list")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"invalid concurrency list",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Running concurrency benchmark (format: %s, levels: %v)...\n", *format, concurrencyLevels)
|
fmt.Printf("Running concurrency benchmark (format: %s, levels: %v)...\n", *format, concurrencyLevels)
|
||||||
suite, err := benchmark.ConcurrencyBenchmark(*sourceDir, *format, concurrencyLevels)
|
suite, err := benchmark.ConcurrencyBenchmark(*sourceDir, *format, concurrencyLevels)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "concurrency benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"concurrency benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
benchmark.PrintBenchmarkSuite(suite)
|
benchmark.PrintSuite(suite)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,9 +117,14 @@ func runFormatBenchmark() error {
|
|||||||
fmt.Printf("Running format benchmark (formats: %v)...\n", formats)
|
fmt.Printf("Running format benchmark (formats: %v)...\n", formats)
|
||||||
suite, err := benchmark.FormatBenchmark(*sourceDir, formats)
|
suite, err := benchmark.FormatBenchmark(*sourceDir, formats)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingCollection, "format benchmark failed")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeProcessing,
|
||||||
|
gibidiutils.CodeProcessingCollection,
|
||||||
|
"format benchmark failed",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
benchmark.PrintBenchmarkSuite(suite)
|
benchmark.PrintSuite(suite)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,16 +143,28 @@ func parseConcurrencyList(list string) ([]int, error) {
|
|||||||
part = strings.TrimSpace(part)
|
part = strings.TrimSpace(part)
|
||||||
var level int
|
var level int
|
||||||
if _, err := fmt.Sscanf(part, "%d", &level); err != nil {
|
if _, err := fmt.Sscanf(part, "%d", &level); err != nil {
|
||||||
return nil, utils.WrapErrorf(err, utils.ErrorTypeValidation, utils.CodeValidationFormat, "invalid concurrency level: %s", part)
|
return nil, gibidiutils.WrapErrorf(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"invalid concurrency level: %s",
|
||||||
|
part,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if level <= 0 {
|
if level <= 0 {
|
||||||
return nil, utils.NewValidationError(utils.CodeValidationFormat, "concurrency level must be positive: "+part)
|
return nil, gibidiutils.NewValidationError(
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"concurrency level must be positive: "+part,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
levels = append(levels, level)
|
levels = append(levels, level)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(levels) == 0 {
|
if len(levels) == 0 {
|
||||||
return nil, utils.NewValidationError(utils.CodeValidationFormat, "no valid concurrency levels found")
|
return nil, gibidiutils.NewValidationError(
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"no valid concurrency levels found",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return levels, nil
|
return levels, nil
|
||||||
|
|||||||
@@ -38,24 +38,24 @@ backpressure:
|
|||||||
# Resource limits for DoS protection and security
|
# Resource limits for DoS protection and security
|
||||||
resourceLimits:
|
resourceLimits:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
||||||
# File processing limits
|
# File processing limits
|
||||||
maxFiles: 10000 # Maximum number of files to process
|
maxFiles: 10000 # Maximum number of files to process
|
||||||
maxTotalSize: 1073741824 # Maximum total size (1GB)
|
maxTotalSize: 1073741824 # Maximum total size (1GB)
|
||||||
|
|
||||||
# Timeout limits (in seconds)
|
# Timeout limits (in seconds)
|
||||||
fileProcessingTimeoutSec: 30 # Timeout for individual file processing
|
fileProcessingTimeoutSec: 30 # Timeout for individual file processing
|
||||||
overallTimeoutSec: 3600 # Overall processing timeout (1 hour)
|
overallTimeoutSec: 3600 # Overall processing timeout (1 hour)
|
||||||
|
|
||||||
# Concurrency limits
|
# Concurrency limits
|
||||||
maxConcurrentReads: 10 # Maximum concurrent file reading operations
|
maxConcurrentReads: 10 # Maximum concurrent file reading operations
|
||||||
|
|
||||||
# Rate limiting (0 = disabled)
|
# Rate limiting (0 = disabled)
|
||||||
rateLimitFilesPerSec: 0 # Files per second rate limit
|
rateLimitFilesPerSec: 0 # Files per second rate limit
|
||||||
|
|
||||||
# Memory limits
|
# Memory limits
|
||||||
hardMemoryLimitMB: 512 # Hard memory limit (512MB)
|
hardMemoryLimitMB: 512 # Hard memory limit (512MB)
|
||||||
|
|
||||||
# Safety features
|
# Safety features
|
||||||
enableGracefulDegradation: true # Enable graceful degradation on resource pressure
|
enableGracefulDegradation: true # Enable graceful degradation on resource pressure
|
||||||
enableResourceMonitoring: true # Enable detailed resource monitoring
|
enableResourceMonitoring: true # Enable detailed resource monitoring
|
||||||
@@ -76,4 +76,4 @@ resourceLimits:
|
|||||||
# filePatterns:
|
# filePatterns:
|
||||||
# - "*.go"
|
# - "*.go"
|
||||||
# - "*.py"
|
# - "*.py"
|
||||||
# - "*.js"
|
# - "*.js"
|
||||||
|
|||||||
@@ -58,4 +58,4 @@ const (
|
|||||||
MinHardMemoryLimitMB = 64
|
MinHardMemoryLimitMB = 64
|
||||||
// MaxHardMemoryLimitMB is the maximum hard memory limit (8192MB = 8GB).
|
// MaxHardMemoryLimitMB is the maximum hard memory limit (8192MB = 8GB).
|
||||||
MaxHardMemoryLimitMB = 8192
|
MaxHardMemoryLimitMB = 8192
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -154,4 +154,4 @@ func GetEnableGracefulDegradation() bool {
|
|||||||
// GetEnableResourceMonitoring returns whether resource monitoring is enabled.
|
// GetEnableResourceMonitoring returns whether resource monitoring is enabled.
|
||||||
func GetEnableResourceMonitoring() bool {
|
func GetEnableResourceMonitoring() bool {
|
||||||
return viper.GetBool("resourceLimits.enableResourceMonitoring")
|
return viper.GetBool("resourceLimits.enableResourceMonitoring")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,13 +1,15 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"flag"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LoadConfig reads configuration from a YAML file.
|
// LoadConfig reads configuration from a YAML file.
|
||||||
@@ -15,13 +17,18 @@ import (
|
|||||||
// 1. $XDG_CONFIG_HOME/gibidify/config.yaml
|
// 1. $XDG_CONFIG_HOME/gibidify/config.yaml
|
||||||
// 2. $HOME/.config/gibidify/config.yaml
|
// 2. $HOME/.config/gibidify/config.yaml
|
||||||
// 3. The current directory as fallback.
|
// 3. The current directory as fallback.
|
||||||
|
//
|
||||||
|
// Note: LoadConfig relies on isRunningTest() which requires the testing package
|
||||||
|
// to have registered its flags (e.g., via flag.Parse() or during test initialization).
|
||||||
|
// If called too early (e.g., from init() or before TestMain), test detection may not work reliably.
|
||||||
|
// For explicit control, use SetRunningInTest() before calling LoadConfig.
|
||||||
func LoadConfig() {
|
func LoadConfig() {
|
||||||
viper.SetConfigName("config")
|
viper.SetConfigName("config")
|
||||||
viper.SetConfigType("yaml")
|
viper.SetConfigType("yaml")
|
||||||
|
|
||||||
if xdgConfig := os.Getenv("XDG_CONFIG_HOME"); xdgConfig != "" {
|
if xdgConfig := os.Getenv("XDG_CONFIG_HOME"); xdgConfig != "" {
|
||||||
// Validate XDG_CONFIG_HOME for path traversal attempts
|
// Validate XDG_CONFIG_HOME for path traversal attempts
|
||||||
if err := utils.ValidateConfigPath(xdgConfig); err != nil {
|
if err := gibidiutils.ValidateConfigPath(xdgConfig); err != nil {
|
||||||
logrus.Warnf("Invalid XDG_CONFIG_HOME path, using default config: %v", err)
|
logrus.Warnf("Invalid XDG_CONFIG_HOME path, using default config: %v", err)
|
||||||
} else {
|
} else {
|
||||||
configPath := filepath.Join(xdgConfig, "gibidify")
|
configPath := filepath.Join(xdgConfig, "gibidify")
|
||||||
@@ -37,7 +44,14 @@ func LoadConfig() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := viper.ReadInConfig(); err != nil {
|
if err := viper.ReadInConfig(); err != nil {
|
||||||
logrus.Infof("Config file not found, using default values: %v", err)
|
// Suppress this info-level log when running tests.
|
||||||
|
// Prefer an explicit test flag (SetRunningInTest) but fall back to runtime detection.
|
||||||
|
if runningInTest.Load() || isRunningTest() {
|
||||||
|
// Keep a debug-level record so tests that enable debug can still see it.
|
||||||
|
logrus.Debugf("Config file not found (tests): %v", err)
|
||||||
|
} else {
|
||||||
|
logrus.Infof("Config file not found, using default values: %v", err)
|
||||||
|
}
|
||||||
setDefaultConfig()
|
setDefaultConfig()
|
||||||
} else {
|
} else {
|
||||||
logrus.Infof("Using config file: %s", viper.ConfigFileUsed())
|
logrus.Infof("Using config file: %s", viper.ConfigFileUsed())
|
||||||
@@ -87,4 +101,31 @@ func setDefaultConfig() {
|
|||||||
viper.SetDefault("resourceLimits.hardMemoryLimitMB", DefaultHardMemoryLimitMB)
|
viper.SetDefault("resourceLimits.hardMemoryLimitMB", DefaultHardMemoryLimitMB)
|
||||||
viper.SetDefault("resourceLimits.enableGracefulDegradation", true)
|
viper.SetDefault("resourceLimits.enableGracefulDegradation", true)
|
||||||
viper.SetDefault("resourceLimits.enableResourceMonitoring", true)
|
viper.SetDefault("resourceLimits.enableResourceMonitoring", true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var runningInTest atomic.Bool
|
||||||
|
|
||||||
|
// SetRunningInTest allows tests to explicitly indicate they are running under `go test`.
|
||||||
|
// Call this from TestMain in tests to suppress noisy info logs while still allowing
|
||||||
|
// debug-level output for tests that enable it.
|
||||||
|
func SetRunningInTest(b bool) {
|
||||||
|
runningInTest.Store(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRunningTest attempts to detect if the binary is running under `go test`.
|
||||||
|
// Prefer checking for standard test flags registered by the testing package.
|
||||||
|
// This is reliable when `go test` initializes the flag set.
|
||||||
|
//
|
||||||
|
// IMPORTANT: This function relies on flag.Lookup which returns nil if the testing
|
||||||
|
// package hasn't registered test flags yet. Callers must invoke this after flag
|
||||||
|
// parsing (or test flag registration) has occurred. If invoked too early (e.g.,
|
||||||
|
// from init() or early in TestMain before flags are parsed), detection will fail.
|
||||||
|
// For explicit control, use SetRunningInTest() instead.
|
||||||
|
func isRunningTest() bool {
|
||||||
|
// Look for the well-known test flags created by the testing package.
|
||||||
|
// If any are present in the flag registry, we're running under `go test`.
|
||||||
|
if flag.Lookup("test.v") != nil || flag.Lookup("test.run") != nil || flag.Lookup("test.bench") != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -79,15 +79,15 @@ func TestLoadConfigWithValidation(t *testing.T) {
|
|||||||
configContent := `
|
configContent := `
|
||||||
fileSizeLimit: 100
|
fileSizeLimit: 100
|
||||||
ignoreDirectories:
|
ignoreDirectories:
|
||||||
- node_modules
|
- node_modules
|
||||||
- ""
|
- ""
|
||||||
- .git
|
- .git
|
||||||
`
|
`
|
||||||
|
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
configFile := tempDir + "/config.yaml"
|
configFile := tempDir + "/config.yaml"
|
||||||
|
|
||||||
err := os.WriteFile(configFile, []byte(configContent), 0o644)
|
err := os.WriteFile(configFile, []byte(configContent), 0o600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to write config file: %v", err)
|
t.Fatalf("Failed to write config file: %v", err)
|
||||||
}
|
}
|
||||||
@@ -104,7 +104,10 @@ ignoreDirectories:
|
|||||||
t.Errorf("Expected default file size limit after validation failure, got %d", config.GetFileSizeLimit())
|
t.Errorf("Expected default file size limit after validation failure, got %d", config.GetFileSizeLimit())
|
||||||
}
|
}
|
||||||
if containsString(config.GetIgnoredDirectories(), "") {
|
if containsString(config.GetIgnoredDirectories(), "") {
|
||||||
t.Errorf("Expected ignored directories not to contain empty string after validation failure, got %v", config.GetIgnoredDirectories())
|
t.Errorf(
|
||||||
|
"Expected ignored directories not to contain empty string after validation failure, got %v",
|
||||||
|
config.GetIgnoredDirectories(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,4 +120,4 @@ func containsString(slice []string, item string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,240 +6,532 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ValidateConfig validates the loaded configuration.
|
// validateFileSizeLimit validates the file size limit configuration.
|
||||||
func ValidateConfig() error {
|
func validateFileSizeLimit() []string {
|
||||||
var validationErrors []string
|
var errors []string
|
||||||
|
|
||||||
// Validate file size limit
|
|
||||||
fileSizeLimit := viper.GetInt64("fileSizeLimit")
|
fileSizeLimit := viper.GetInt64("fileSizeLimit")
|
||||||
if fileSizeLimit < MinFileSizeLimit {
|
if fileSizeLimit < MinFileSizeLimit {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileSizeLimit (%d) is below minimum (%d)", fileSizeLimit, MinFileSizeLimit))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("fileSizeLimit (%d) is below minimum (%d)", fileSizeLimit, MinFileSizeLimit),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if fileSizeLimit > MaxFileSizeLimit {
|
if fileSizeLimit > MaxFileSizeLimit {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileSizeLimit (%d) exceeds maximum (%d)", fileSizeLimit, MaxFileSizeLimit))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("fileSizeLimit (%d) exceeds maximum (%d)", fileSizeLimit, MaxFileSizeLimit),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
// Validate ignore directories
|
// validateIgnoreDirectories validates the ignore directories configuration.
|
||||||
|
func validateIgnoreDirectories() []string {
|
||||||
|
var errors []string
|
||||||
ignoreDirectories := viper.GetStringSlice("ignoreDirectories")
|
ignoreDirectories := viper.GetStringSlice("ignoreDirectories")
|
||||||
for i, dir := range ignoreDirectories {
|
for i, dir := range ignoreDirectories {
|
||||||
dir = strings.TrimSpace(dir)
|
dir = strings.TrimSpace(dir)
|
||||||
if dir == "" {
|
if dir == "" {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("ignoreDirectories[%d] is empty", i))
|
errors = append(errors, fmt.Sprintf("ignoreDirectories[%d] is empty", i))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if strings.Contains(dir, "/") {
|
if strings.Contains(dir, "/") {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("ignoreDirectories[%d] (%s) contains path separator - only directory names are allowed", i, dir))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"ignoreDirectories[%d] (%s) contains path separator - only directory names are allowed",
|
||||||
|
i,
|
||||||
|
dir,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(dir, ".") && dir != ".git" && dir != ".vscode" && dir != ".idea" {
|
if strings.HasPrefix(dir, ".") && dir != ".git" && dir != ".vscode" && dir != ".idea" {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("ignoreDirectories[%d] (%s) starts with dot - this may cause unexpected behavior", i, dir))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("ignoreDirectories[%d] (%s) starts with dot - this may cause unexpected behavior", i, dir),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
// Validate supported output formats if configured
|
// validateSupportedFormats validates the supported output formats configuration.
|
||||||
|
func validateSupportedFormats() []string {
|
||||||
|
var errors []string
|
||||||
if viper.IsSet("supportedFormats") {
|
if viper.IsSet("supportedFormats") {
|
||||||
supportedFormats := viper.GetStringSlice("supportedFormats")
|
supportedFormats := viper.GetStringSlice("supportedFormats")
|
||||||
validFormats := map[string]bool{"json": true, "yaml": true, "markdown": true}
|
validFormats := map[string]bool{"json": true, "yaml": true, "markdown": true}
|
||||||
for i, format := range supportedFormats {
|
for i, format := range supportedFormats {
|
||||||
format = strings.ToLower(strings.TrimSpace(format))
|
format = strings.ToLower(strings.TrimSpace(format))
|
||||||
if !validFormats[format] {
|
if !validFormats[format] {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("supportedFormats[%d] (%s) is not a valid format (json, yaml, markdown)", i, format))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("supportedFormats[%d] (%s) is not a valid format (json, yaml, markdown)", i, format),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
// Validate concurrency settings if configured
|
// validateConcurrencySettings validates the concurrency settings configuration.
|
||||||
|
func validateConcurrencySettings() []string {
|
||||||
|
var errors []string
|
||||||
if viper.IsSet("maxConcurrency") {
|
if viper.IsSet("maxConcurrency") {
|
||||||
maxConcurrency := viper.GetInt("maxConcurrency")
|
maxConcurrency := viper.GetInt("maxConcurrency")
|
||||||
if maxConcurrency < 1 {
|
if maxConcurrency < 1 {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("maxConcurrency (%d) must be at least 1", maxConcurrency))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("maxConcurrency (%d) must be at least 1", maxConcurrency),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if maxConcurrency > 100 {
|
if maxConcurrency > 100 {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("maxConcurrency (%d) is unreasonably high (max 100)", maxConcurrency))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("maxConcurrency (%d) is unreasonably high (max 100)", maxConcurrency),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
// Validate file patterns if configured
|
// validateFilePatterns validates the file patterns configuration.
|
||||||
|
func validateFilePatterns() []string {
|
||||||
|
var errors []string
|
||||||
if viper.IsSet("filePatterns") {
|
if viper.IsSet("filePatterns") {
|
||||||
filePatterns := viper.GetStringSlice("filePatterns")
|
filePatterns := viper.GetStringSlice("filePatterns")
|
||||||
for i, pattern := range filePatterns {
|
for i, pattern := range filePatterns {
|
||||||
pattern = strings.TrimSpace(pattern)
|
pattern = strings.TrimSpace(pattern)
|
||||||
if pattern == "" {
|
if pattern == "" {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("filePatterns[%d] is empty", i))
|
errors = append(errors, fmt.Sprintf("filePatterns[%d] is empty", i))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Basic validation - patterns should contain at least one alphanumeric character
|
// Basic validation - patterns should contain at least one alphanumeric character
|
||||||
if !strings.ContainsAny(pattern, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") {
|
if !strings.ContainsAny(pattern, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("filePatterns[%d] (%s) appears to be invalid", i, pattern))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("filePatterns[%d] (%s) appears to be invalid", i, pattern),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
// Validate FileTypeRegistry configuration
|
// validateFileTypes validates the FileTypeRegistry configuration.
|
||||||
if viper.IsSet("fileTypes.customImageExtensions") {
|
// validateCustomImageExtensions validates custom image extensions configuration.
|
||||||
customImages := viper.GetStringSlice("fileTypes.customImageExtensions")
|
func validateCustomImageExtensions() []string {
|
||||||
for i, ext := range customImages {
|
var errors []string
|
||||||
ext = strings.TrimSpace(ext)
|
if !viper.IsSet("fileTypes.customImageExtensions") {
|
||||||
if ext == "" {
|
return errors
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customImageExtensions[%d] is empty", i))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(ext, ".") {
|
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customImageExtensions[%d] (%s) must start with a dot", i, ext))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("fileTypes.customBinaryExtensions") {
|
customImages := viper.GetStringSlice("fileTypes.customImageExtensions")
|
||||||
customBinary := viper.GetStringSlice("fileTypes.customBinaryExtensions")
|
for i, ext := range customImages {
|
||||||
for i, ext := range customBinary {
|
ext = strings.TrimSpace(ext)
|
||||||
ext = strings.TrimSpace(ext)
|
if ext == "" {
|
||||||
if ext == "" {
|
errors = append(
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customBinaryExtensions[%d] is empty", i))
|
errors,
|
||||||
continue
|
fmt.Sprintf("fileTypes.customImageExtensions[%d] is empty", i),
|
||||||
}
|
)
|
||||||
if !strings.HasPrefix(ext, ".") {
|
continue
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customBinaryExtensions[%d] (%s) must start with a dot", i, ext))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if !strings.HasPrefix(ext, ".") {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("fileTypes.customImageExtensions[%d] (%s) must start with a dot", i, ext),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateCustomBinaryExtensions validates custom binary extensions configuration.
|
||||||
|
func validateCustomBinaryExtensions() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("fileTypes.customBinaryExtensions") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("fileTypes.customLanguages") {
|
customBinary := viper.GetStringSlice("fileTypes.customBinaryExtensions")
|
||||||
customLangs := viper.GetStringMapString("fileTypes.customLanguages")
|
for i, ext := range customBinary {
|
||||||
for ext, lang := range customLangs {
|
ext = strings.TrimSpace(ext)
|
||||||
ext = strings.TrimSpace(ext)
|
if ext == "" {
|
||||||
lang = strings.TrimSpace(lang)
|
errors = append(
|
||||||
if ext == "" {
|
errors,
|
||||||
validationErrors = append(validationErrors, "fileTypes.customLanguages contains empty extension key")
|
fmt.Sprintf("fileTypes.customBinaryExtensions[%d] is empty", i),
|
||||||
continue
|
)
|
||||||
}
|
continue
|
||||||
if !strings.HasPrefix(ext, ".") {
|
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customLanguages extension (%s) must start with a dot", ext))
|
|
||||||
}
|
|
||||||
if lang == "" {
|
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("fileTypes.customLanguages[%s] has empty language value", ext))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if !strings.HasPrefix(ext, ".") {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("fileTypes.customBinaryExtensions[%d] (%s) must start with a dot", i, ext),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateCustomLanguages validates custom languages configuration.
|
||||||
|
func validateCustomLanguages() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("fileTypes.customLanguages") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate back-pressure configuration
|
customLangs := viper.GetStringMapString("fileTypes.customLanguages")
|
||||||
if viper.IsSet("backpressure.maxPendingFiles") {
|
for ext, lang := range customLangs {
|
||||||
maxPendingFiles := viper.GetInt("backpressure.maxPendingFiles")
|
ext = strings.TrimSpace(ext)
|
||||||
if maxPendingFiles < 1 {
|
lang = strings.TrimSpace(lang)
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingFiles (%d) must be at least 1", maxPendingFiles))
|
if ext == "" {
|
||||||
|
errors = append(errors, "fileTypes.customLanguages contains empty extension key")
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
if maxPendingFiles > 100000 {
|
if !strings.HasPrefix(ext, ".") {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingFiles (%d) is unreasonably high (max 100000)", maxPendingFiles))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("fileTypes.customLanguages extension (%s) must start with a dot", ext),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
if lang == "" {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("fileTypes.customLanguages[%s] has empty language value", ext),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateFileTypes validates the FileTypeRegistry configuration.
|
||||||
|
func validateFileTypes() []string {
|
||||||
|
var errors []string
|
||||||
|
errors = append(errors, validateCustomImageExtensions()...)
|
||||||
|
errors = append(errors, validateCustomBinaryExtensions()...)
|
||||||
|
errors = append(errors, validateCustomLanguages()...)
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateBackpressureConfig validates the back-pressure configuration.
|
||||||
|
// validateBackpressureMaxPendingFiles validates max pending files configuration.
|
||||||
|
func validateBackpressureMaxPendingFiles() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("backpressure.maxPendingFiles") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("backpressure.maxPendingWrites") {
|
maxPendingFiles := viper.GetInt("backpressure.maxPendingFiles")
|
||||||
maxPendingWrites := viper.GetInt("backpressure.maxPendingWrites")
|
if maxPendingFiles < 1 {
|
||||||
if maxPendingWrites < 1 {
|
errors = append(
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingWrites (%d) must be at least 1", maxPendingWrites))
|
errors,
|
||||||
}
|
fmt.Sprintf("backpressure.maxPendingFiles (%d) must be at least 1", maxPendingFiles),
|
||||||
if maxPendingWrites > 10000 {
|
)
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxPendingWrites (%d) is unreasonably high (max 10000)", maxPendingWrites))
|
}
|
||||||
}
|
if maxPendingFiles > 100000 {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("backpressure.maxPendingFiles (%d) is unreasonably high (max 100000)", maxPendingFiles),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateBackpressureMaxPendingWrites validates max pending writes configuration.
|
||||||
|
func validateBackpressureMaxPendingWrites() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("backpressure.maxPendingWrites") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("backpressure.maxMemoryUsage") {
|
maxPendingWrites := viper.GetInt("backpressure.maxPendingWrites")
|
||||||
maxMemoryUsage := viper.GetInt64("backpressure.maxMemoryUsage")
|
if maxPendingWrites < 1 {
|
||||||
if maxMemoryUsage < 1048576 { // 1MB minimum
|
errors = append(
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxMemoryUsage (%d) must be at least 1MB (1048576 bytes)", maxMemoryUsage))
|
errors,
|
||||||
}
|
fmt.Sprintf("backpressure.maxPendingWrites (%d) must be at least 1", maxPendingWrites),
|
||||||
if maxMemoryUsage > 10737418240 { // 10GB maximum
|
)
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.maxMemoryUsage (%d) is unreasonably high (max 10GB)", maxMemoryUsage))
|
}
|
||||||
}
|
if maxPendingWrites > 10000 {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("backpressure.maxPendingWrites (%d) is unreasonably high (max 10000)", maxPendingWrites),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateBackpressureMaxMemoryUsage validates max memory usage configuration.
|
||||||
|
func validateBackpressureMaxMemoryUsage() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("backpressure.maxMemoryUsage") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("backpressure.memoryCheckInterval") {
|
maxMemoryUsage := viper.GetInt64("backpressure.maxMemoryUsage")
|
||||||
interval := viper.GetInt("backpressure.memoryCheckInterval")
|
if maxMemoryUsage < 1048576 { // 1MB minimum
|
||||||
if interval < 1 {
|
errors = append(
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.memoryCheckInterval (%d) must be at least 1", interval))
|
errors,
|
||||||
}
|
fmt.Sprintf("backpressure.maxMemoryUsage (%d) must be at least 1MB (1048576 bytes)", maxMemoryUsage),
|
||||||
if interval > 100000 {
|
)
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("backpressure.memoryCheckInterval (%d) is unreasonably high (max 100000)", interval))
|
}
|
||||||
}
|
if maxMemoryUsage > 104857600 { // 100MB maximum
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("backpressure.maxMemoryUsage (%d) is unreasonably high (max 100MB)", maxMemoryUsage),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateBackpressureMemoryCheckInterval validates memory check interval configuration.
|
||||||
|
func validateBackpressureMemoryCheckInterval() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("backpressure.memoryCheckInterval") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate resource limits configuration
|
interval := viper.GetInt("backpressure.memoryCheckInterval")
|
||||||
if viper.IsSet("resourceLimits.maxFiles") {
|
if interval < 1 {
|
||||||
maxFiles := viper.GetInt("resourceLimits.maxFiles")
|
errors = append(
|
||||||
if maxFiles < MinMaxFiles {
|
errors,
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxFiles (%d) must be at least %d", maxFiles, MinMaxFiles))
|
fmt.Sprintf("backpressure.memoryCheckInterval (%d) must be at least 1", interval),
|
||||||
}
|
)
|
||||||
if maxFiles > MaxMaxFiles {
|
}
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxFiles (%d) exceeds maximum (%d)", maxFiles, MaxMaxFiles))
|
if interval > 100000 {
|
||||||
}
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("backpressure.memoryCheckInterval (%d) is unreasonably high (max 100000)", interval),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateBackpressureConfig validates the back-pressure configuration.
|
||||||
|
func validateBackpressureConfig() []string {
|
||||||
|
var errors []string
|
||||||
|
errors = append(errors, validateBackpressureMaxPendingFiles()...)
|
||||||
|
errors = append(errors, validateBackpressureMaxPendingWrites()...)
|
||||||
|
errors = append(errors, validateBackpressureMaxMemoryUsage()...)
|
||||||
|
errors = append(errors, validateBackpressureMemoryCheckInterval()...)
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateResourceLimits validates the resource limits configuration.
|
||||||
|
// validateResourceLimitsMaxFiles validates max files configuration.
|
||||||
|
func validateResourceLimitsMaxFiles() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("resourceLimits.maxFiles") {
|
||||||
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("resourceLimits.maxTotalSize") {
|
maxFiles := viper.GetInt("resourceLimits.maxFiles")
|
||||||
maxTotalSize := viper.GetInt64("resourceLimits.maxTotalSize")
|
if maxFiles < MinMaxFiles {
|
||||||
if maxTotalSize < MinMaxTotalSize {
|
errors = append(
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxTotalSize (%d) must be at least %d", maxTotalSize, MinMaxTotalSize))
|
errors,
|
||||||
}
|
fmt.Sprintf("resourceLimits.maxFiles (%d) must be at least %d", maxFiles, MinMaxFiles),
|
||||||
if maxTotalSize > MaxMaxTotalSize {
|
)
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxTotalSize (%d) exceeds maximum (%d)", maxTotalSize, MaxMaxTotalSize))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if maxFiles > MaxMaxFiles {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("resourceLimits.maxFiles (%d) exceeds maximum (%d)", maxFiles, MaxMaxFiles),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateResourceLimitsMaxTotalSize validates max total size configuration.
|
||||||
|
func validateResourceLimitsMaxTotalSize() []string {
|
||||||
|
var errors []string
|
||||||
|
if !viper.IsSet("resourceLimits.maxTotalSize") {
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
maxTotalSize := viper.GetInt64("resourceLimits.maxTotalSize")
|
||||||
|
if maxTotalSize < MinMaxTotalSize {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("resourceLimits.maxTotalSize (%d) must be at least %d", maxTotalSize, MinMaxTotalSize),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if maxTotalSize > MaxMaxTotalSize {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("resourceLimits.maxTotalSize (%d) exceeds maximum (%d)", maxTotalSize, MaxMaxTotalSize),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateResourceLimitsTimeouts validates timeout configurations.
|
||||||
|
func validateResourceLimitsTimeouts() []string {
|
||||||
|
var errors []string
|
||||||
|
|
||||||
if viper.IsSet("resourceLimits.fileProcessingTimeoutSec") {
|
if viper.IsSet("resourceLimits.fileProcessingTimeoutSec") {
|
||||||
timeout := viper.GetInt("resourceLimits.fileProcessingTimeoutSec")
|
timeout := viper.GetInt("resourceLimits.fileProcessingTimeoutSec")
|
||||||
if timeout < MinFileProcessingTimeoutSec {
|
if timeout < MinFileProcessingTimeoutSec {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.fileProcessingTimeoutSec (%d) must be at least %d", timeout, MinFileProcessingTimeoutSec))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.fileProcessingTimeoutSec (%d) must be at least %d",
|
||||||
|
timeout,
|
||||||
|
MinFileProcessingTimeoutSec,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if timeout > MaxFileProcessingTimeoutSec {
|
if timeout > MaxFileProcessingTimeoutSec {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.fileProcessingTimeoutSec (%d) exceeds maximum (%d)", timeout, MaxFileProcessingTimeoutSec))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.fileProcessingTimeoutSec (%d) exceeds maximum (%d)",
|
||||||
|
timeout,
|
||||||
|
MaxFileProcessingTimeoutSec,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("resourceLimits.overallTimeoutSec") {
|
if viper.IsSet("resourceLimits.overallTimeoutSec") {
|
||||||
timeout := viper.GetInt("resourceLimits.overallTimeoutSec")
|
timeout := viper.GetInt("resourceLimits.overallTimeoutSec")
|
||||||
if timeout < MinOverallTimeoutSec {
|
if timeout < MinOverallTimeoutSec {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) must be at least %d", timeout, MinOverallTimeoutSec))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) must be at least %d", timeout, MinOverallTimeoutSec),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if timeout > MaxOverallTimeoutSec {
|
if timeout > MaxOverallTimeoutSec {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.overallTimeoutSec (%d) exceeds maximum (%d)", timeout, MaxOverallTimeoutSec))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.overallTimeoutSec (%d) exceeds maximum (%d)",
|
||||||
|
timeout,
|
||||||
|
MaxOverallTimeoutSec,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateResourceLimitsConcurrency validates concurrency configurations.
|
||||||
|
func validateResourceLimitsConcurrency() []string {
|
||||||
|
var errors []string
|
||||||
|
|
||||||
if viper.IsSet("resourceLimits.maxConcurrentReads") {
|
if viper.IsSet("resourceLimits.maxConcurrentReads") {
|
||||||
maxReads := viper.GetInt("resourceLimits.maxConcurrentReads")
|
maxReads := viper.GetInt("resourceLimits.maxConcurrentReads")
|
||||||
if maxReads < MinMaxConcurrentReads {
|
if maxReads < MinMaxConcurrentReads {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) must be at least %d", maxReads, MinMaxConcurrentReads))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.maxConcurrentReads (%d) must be at least %d",
|
||||||
|
maxReads,
|
||||||
|
MinMaxConcurrentReads,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if maxReads > MaxMaxConcurrentReads {
|
if maxReads > MaxMaxConcurrentReads {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.maxConcurrentReads (%d) exceeds maximum (%d)", maxReads, MaxMaxConcurrentReads))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.maxConcurrentReads (%d) exceeds maximum (%d)",
|
||||||
|
maxReads,
|
||||||
|
MaxMaxConcurrentReads,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("resourceLimits.rateLimitFilesPerSec") {
|
if viper.IsSet("resourceLimits.rateLimitFilesPerSec") {
|
||||||
rateLimit := viper.GetInt("resourceLimits.rateLimitFilesPerSec")
|
rateLimit := viper.GetInt("resourceLimits.rateLimitFilesPerSec")
|
||||||
if rateLimit < MinRateLimitFilesPerSec {
|
if rateLimit < MinRateLimitFilesPerSec {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) must be at least %d", rateLimit, MinRateLimitFilesPerSec))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.rateLimitFilesPerSec (%d) must be at least %d",
|
||||||
|
rateLimit,
|
||||||
|
MinRateLimitFilesPerSec,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if rateLimit > MaxRateLimitFilesPerSec {
|
if rateLimit > MaxRateLimitFilesPerSec {
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.rateLimitFilesPerSec (%d) exceeds maximum (%d)", rateLimit, MaxRateLimitFilesPerSec))
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.rateLimitFilesPerSec (%d) exceeds maximum (%d)",
|
||||||
|
rateLimit,
|
||||||
|
MaxRateLimitFilesPerSec,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if viper.IsSet("resourceLimits.hardMemoryLimitMB") {
|
return errors
|
||||||
memLimit := viper.GetInt("resourceLimits.hardMemoryLimitMB")
|
}
|
||||||
if memLimit < MinHardMemoryLimitMB {
|
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) must be at least %d", memLimit, MinHardMemoryLimitMB))
|
// validateResourceLimitsMemory validates memory limit configuration.
|
||||||
}
|
func validateResourceLimitsMemory() []string {
|
||||||
if memLimit > MaxHardMemoryLimitMB {
|
var errors []string
|
||||||
validationErrors = append(validationErrors, fmt.Sprintf("resourceLimits.hardMemoryLimitMB (%d) exceeds maximum (%d)", memLimit, MaxHardMemoryLimitMB))
|
if !viper.IsSet("resourceLimits.hardMemoryLimitMB") {
|
||||||
}
|
return errors
|
||||||
}
|
}
|
||||||
|
|
||||||
|
memLimit := viper.GetInt("resourceLimits.hardMemoryLimitMB")
|
||||||
|
if memLimit < MinHardMemoryLimitMB {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.hardMemoryLimitMB (%d) must be at least %d",
|
||||||
|
memLimit,
|
||||||
|
MinHardMemoryLimitMB,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if memLimit > MaxHardMemoryLimitMB {
|
||||||
|
errors = append(
|
||||||
|
errors,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"resourceLimits.hardMemoryLimitMB (%d) exceeds maximum (%d)",
|
||||||
|
memLimit,
|
||||||
|
MaxHardMemoryLimitMB,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateResourceLimits validates the resource limits configuration.
|
||||||
|
func validateResourceLimits() []string {
|
||||||
|
var errors []string
|
||||||
|
errors = append(errors, validateResourceLimitsMaxFiles()...)
|
||||||
|
errors = append(errors, validateResourceLimitsMaxTotalSize()...)
|
||||||
|
errors = append(errors, validateResourceLimitsTimeouts()...)
|
||||||
|
errors = append(errors, validateResourceLimitsConcurrency()...)
|
||||||
|
errors = append(errors, validateResourceLimitsMemory()...)
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateConfig validates the loaded configuration.
|
||||||
|
func ValidateConfig() error {
|
||||||
|
var validationErrors []string
|
||||||
|
|
||||||
|
// Collect validation errors from all validation helpers
|
||||||
|
validationErrors = append(validationErrors, validateFileSizeLimit()...)
|
||||||
|
validationErrors = append(validationErrors, validateIgnoreDirectories()...)
|
||||||
|
validationErrors = append(validationErrors, validateSupportedFormats()...)
|
||||||
|
validationErrors = append(validationErrors, validateConcurrencySettings()...)
|
||||||
|
validationErrors = append(validationErrors, validateFilePatterns()...)
|
||||||
|
validationErrors = append(validationErrors, validateFileTypes()...)
|
||||||
|
validationErrors = append(validationErrors, validateBackpressureConfig()...)
|
||||||
|
validationErrors = append(validationErrors, validateResourceLimits()...)
|
||||||
|
|
||||||
if len(validationErrors) > 0 {
|
if len(validationErrors) > 0 {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeConfiguration,
|
gibidiutils.ErrorTypeConfiguration,
|
||||||
utils.CodeConfigValidation,
|
gibidiutils.CodeConfigValidation,
|
||||||
"configuration validation failed: "+strings.Join(validationErrors, "; "),
|
"configuration validation failed: "+strings.Join(validationErrors, "; "),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{"validation_errors": validationErrors},
|
map[string]interface{}{"validation_errors": validationErrors},
|
||||||
@@ -253,9 +545,9 @@ func ValidateConfig() error {
|
|||||||
func ValidateFileSize(size int64) error {
|
func ValidateFileSize(size int64) error {
|
||||||
limit := GetFileSizeLimit()
|
limit := GetFileSizeLimit()
|
||||||
if size > limit {
|
if size > limit {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeValidationSize,
|
gibidiutils.CodeValidationSize,
|
||||||
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", size, limit),
|
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", size, limit),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{"file_size": size, "size_limit": limit},
|
map[string]interface{}{"file_size": size, "size_limit": limit},
|
||||||
@@ -267,9 +559,9 @@ func ValidateFileSize(size int64) error {
|
|||||||
// ValidateOutputFormat checks if an output format is valid.
|
// ValidateOutputFormat checks if an output format is valid.
|
||||||
func ValidateOutputFormat(format string) error {
|
func ValidateOutputFormat(format string) error {
|
||||||
if !IsValidFormat(format) {
|
if !IsValidFormat(format) {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeValidationFormat,
|
gibidiutils.CodeValidationFormat,
|
||||||
fmt.Sprintf("unsupported output format: %s (supported: json, yaml, markdown)", format),
|
fmt.Sprintf("unsupported output format: %s (supported: json, yaml, markdown)", format),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{"format": format},
|
map[string]interface{}{"format": format},
|
||||||
@@ -281,9 +573,9 @@ func ValidateOutputFormat(format string) error {
|
|||||||
// ValidateConcurrency checks if a concurrency level is valid.
|
// ValidateConcurrency checks if a concurrency level is valid.
|
||||||
func ValidateConcurrency(concurrency int) error {
|
func ValidateConcurrency(concurrency int) error {
|
||||||
if concurrency < 1 {
|
if concurrency < 1 {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeValidationFormat,
|
gibidiutils.CodeValidationFormat,
|
||||||
fmt.Sprintf("concurrency (%d) must be at least 1", concurrency),
|
fmt.Sprintf("concurrency (%d) must be at least 1", concurrency),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{"concurrency": concurrency},
|
map[string]interface{}{"concurrency": concurrency},
|
||||||
@@ -293,9 +585,9 @@ func ValidateConcurrency(concurrency int) error {
|
|||||||
if viper.IsSet("maxConcurrency") {
|
if viper.IsSet("maxConcurrency") {
|
||||||
maxConcurrency := GetMaxConcurrency()
|
maxConcurrency := GetMaxConcurrency()
|
||||||
if concurrency > maxConcurrency {
|
if concurrency > maxConcurrency {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeValidationFormat,
|
gibidiutils.CodeValidationFormat,
|
||||||
fmt.Sprintf("concurrency (%d) exceeds maximum (%d)", concurrency, maxConcurrency),
|
fmt.Sprintf("concurrency (%d) exceeds maximum (%d)", concurrency, maxConcurrency),
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{"concurrency": concurrency, "max_concurrency": maxConcurrency},
|
map[string]interface{}{"concurrency": concurrency, "max_concurrency": maxConcurrency},
|
||||||
@@ -304,4 +596,4 @@ func ValidateConcurrency(concurrency int) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/config"
|
"github.com/ivuorinen/gibidify/config"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestValidateConfig tests the configuration validation functionality.
|
// TestValidateConfig tests the configuration validation functionality.
|
||||||
@@ -112,21 +113,19 @@ func TestValidateConfig(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check that it's a structured error
|
// Check that it's a structured error
|
||||||
var structErr *utils.StructuredError
|
var structErr *gibidiutils.StructuredError
|
||||||
if !errorAs(err, &structErr) {
|
if !errorAs(err, &structErr) {
|
||||||
t.Errorf("Expected structured error, got %T", err)
|
t.Errorf("Expected structured error, got %T", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if structErr.Type != utils.ErrorTypeConfiguration {
|
if structErr.Type != gibidiutils.ErrorTypeConfiguration {
|
||||||
t.Errorf("Expected error type %v, got %v", utils.ErrorTypeConfiguration, structErr.Type)
|
t.Errorf("Expected error type %v, got %v", gibidiutils.ErrorTypeConfiguration, structErr.Type)
|
||||||
}
|
}
|
||||||
if structErr.Code != utils.CodeConfigValidation {
|
if structErr.Code != gibidiutils.CodeConfigValidation {
|
||||||
t.Errorf("Expected error code %v, got %v", utils.CodeConfigValidation, structErr.Code)
|
t.Errorf("Expected error code %v, got %v", gibidiutils.CodeConfigValidation, structErr.Code)
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Expected no error but got: %v", err)
|
|
||||||
}
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
t.Errorf("Expected no error but got: %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -235,11 +234,12 @@ func errorAs(err error, target interface{}) bool {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if structErr, ok := err.(*utils.StructuredError); ok {
|
var structErr *gibidiutils.StructuredError
|
||||||
if ptr, ok := target.(**utils.StructuredError); ok {
|
if errors.As(err, &structErr) {
|
||||||
|
if ptr, ok := target.(**gibidiutils.StructuredError); ok {
|
||||||
*ptr = structErr
|
*ptr = structErr
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package fileproc
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"math"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/config"
|
"github.com/ivuorinen/gibidify/config"
|
||||||
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BackpressureManager manages memory usage and applies back-pressure when needed.
|
// BackpressureManager manages memory usage and applies back-pressure when needed.
|
||||||
@@ -59,21 +61,22 @@ func (bp *BackpressureManager) CreateChannels() (chan string, chan WriteRequest)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ShouldApplyBackpressure checks if back-pressure should be applied.
|
// ShouldApplyBackpressure checks if back-pressure should be applied.
|
||||||
func (bp *BackpressureManager) ShouldApplyBackpressure(ctx context.Context) bool {
|
func (bp *BackpressureManager) ShouldApplyBackpressure(_ context.Context) bool {
|
||||||
if !bp.enabled {
|
if !bp.enabled {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if we should evaluate memory usage
|
// Check if we should evaluate memory usage
|
||||||
filesProcessed := atomic.AddInt64(&bp.filesProcessed, 1)
|
filesProcessed := atomic.AddInt64(&bp.filesProcessed, 1)
|
||||||
if int(filesProcessed)%bp.memoryCheckInterval != 0 {
|
// Avoid divide by zero - if interval is 0, check every file
|
||||||
|
if bp.memoryCheckInterval > 0 && int(filesProcessed)%bp.memoryCheckInterval != 0 {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get current memory usage
|
// Get current memory usage
|
||||||
var m runtime.MemStats
|
var m runtime.MemStats
|
||||||
runtime.ReadMemStats(&m)
|
runtime.ReadMemStats(&m)
|
||||||
currentMemory := int64(m.Alloc)
|
currentMemory := gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, math.MaxInt64)
|
||||||
|
|
||||||
bp.mu.Lock()
|
bp.mu.Lock()
|
||||||
defer bp.mu.Unlock()
|
defer bp.mu.Unlock()
|
||||||
@@ -133,7 +136,7 @@ func (bp *BackpressureManager) GetStats() BackpressureStats {
|
|||||||
return BackpressureStats{
|
return BackpressureStats{
|
||||||
Enabled: bp.enabled,
|
Enabled: bp.enabled,
|
||||||
FilesProcessed: atomic.LoadInt64(&bp.filesProcessed),
|
FilesProcessed: atomic.LoadInt64(&bp.filesProcessed),
|
||||||
CurrentMemoryUsage: int64(m.Alloc),
|
CurrentMemoryUsage: gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, math.MaxInt64),
|
||||||
MaxMemoryUsage: bp.maxMemoryUsage,
|
MaxMemoryUsage: bp.maxMemoryUsage,
|
||||||
MemoryWarningActive: bp.memoryWarningLogged,
|
MemoryWarningActive: bp.memoryWarningLogged,
|
||||||
LastMemoryCheck: bp.lastMemoryCheck,
|
LastMemoryCheck: bp.lastMemoryCheck,
|
||||||
@@ -160,8 +163,8 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file channel is getting full (>90% capacity)
|
// Check if file channel is getting full (>=90% capacity)
|
||||||
if len(fileCh) > bp.maxPendingFiles*9/10 {
|
if bp.maxPendingFiles > 0 && len(fileCh) >= bp.maxPendingFiles*9/10 {
|
||||||
logrus.Debugf("File channel is %d%% full, waiting for space", len(fileCh)*100/bp.maxPendingFiles)
|
logrus.Debugf("File channel is %d%% full, waiting for space", len(fileCh)*100/bp.maxPendingFiles)
|
||||||
|
|
||||||
// Wait a bit for the channel to drain
|
// Wait a bit for the channel to drain
|
||||||
@@ -172,8 +175,8 @@ func (bp *BackpressureManager) WaitForChannelSpace(ctx context.Context, fileCh c
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if write channel is getting full (>90% capacity)
|
// Check if write channel is getting full (>=90% capacity)
|
||||||
if len(writeCh) > bp.maxPendingWrites*9/10 {
|
if bp.maxPendingWrites > 0 && len(writeCh) >= bp.maxPendingWrites*9/10 {
|
||||||
logrus.Debugf("Write channel is %d%% full, waiting for space", len(writeCh)*100/bp.maxPendingWrites)
|
logrus.Debugf("Write channel is %d%% full, waiting for space", len(writeCh)*100/bp.maxPendingWrites)
|
||||||
|
|
||||||
// Wait a bit for the channel to drain
|
// Wait a bit for the channel to drain
|
||||||
|
|||||||
177
fileproc/backpressure_behavior_test.go
Normal file
177
fileproc/backpressure_behavior_test.go
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
package fileproc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBackpressureManagerShouldApplyBackpressure(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("returns false when disabled", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = false
|
||||||
|
|
||||||
|
shouldApply := bm.ShouldApplyBackpressure(ctx)
|
||||||
|
assert.False(t, shouldApply)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("checks memory at intervals", func(_ *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true
|
||||||
|
bm.memoryCheckInterval = 10
|
||||||
|
|
||||||
|
// Should not check memory on most calls
|
||||||
|
for i := 1; i < 10; i++ {
|
||||||
|
shouldApply := bm.ShouldApplyBackpressure(ctx)
|
||||||
|
// Can't predict result, but shouldn't panic
|
||||||
|
_ = shouldApply
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should check memory on 10th call
|
||||||
|
shouldApply := bm.ShouldApplyBackpressure(ctx)
|
||||||
|
// Result depends on actual memory usage
|
||||||
|
_ = shouldApply
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("detects high memory usage", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true
|
||||||
|
bm.memoryCheckInterval = 1
|
||||||
|
bm.maxMemoryUsage = 1 // Set very low limit to trigger
|
||||||
|
|
||||||
|
shouldApply := bm.ShouldApplyBackpressure(ctx)
|
||||||
|
// Should detect high memory usage
|
||||||
|
assert.True(t, shouldApply)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerApplyBackpressure(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("does nothing when disabled", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = false
|
||||||
|
|
||||||
|
// Use a channel to verify the function returns quickly
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
bm.ApplyBackpressure(ctx)
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Should complete quickly when disabled
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success - function returned
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
t.Fatal("ApplyBackpressure did not return quickly when disabled")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("applies delay when enabled", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true
|
||||||
|
|
||||||
|
// Use a channel to verify the function blocks for some time
|
||||||
|
done := make(chan struct{})
|
||||||
|
started := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
close(started)
|
||||||
|
bm.ApplyBackpressure(ctx)
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for goroutine to start
|
||||||
|
<-started
|
||||||
|
|
||||||
|
// Should NOT complete immediately - verify it blocks for at least 5ms
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatal("ApplyBackpressure returned too quickly when enabled")
|
||||||
|
case <-time.After(5 * time.Millisecond):
|
||||||
|
// Good - it's blocking as expected
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now wait for it to complete (should finish within reasonable time)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success - function eventually returned
|
||||||
|
case <-time.After(500 * time.Millisecond):
|
||||||
|
t.Fatal("ApplyBackpressure did not complete within timeout")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("respects context cancellation", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // Cancel immediately
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
bm.ApplyBackpressure(ctx)
|
||||||
|
duration := time.Since(start)
|
||||||
|
|
||||||
|
// Should return quickly when context is cancelled
|
||||||
|
assert.Less(t, duration, 5*time.Millisecond)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerLogBackpressureInfo(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true // Ensure enabled so filesProcessed is incremented
|
||||||
|
|
||||||
|
// Apply some operations
|
||||||
|
ctx := context.Background()
|
||||||
|
bm.ShouldApplyBackpressure(ctx)
|
||||||
|
bm.ApplyBackpressure(ctx)
|
||||||
|
|
||||||
|
// This should not panic
|
||||||
|
bm.LogBackpressureInfo()
|
||||||
|
|
||||||
|
stats := bm.GetStats()
|
||||||
|
assert.Greater(t, stats.FilesProcessed, int64(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerMemoryLimiting(t *testing.T) {
|
||||||
|
t.Run("triggers on low memory limit", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true
|
||||||
|
bm.memoryCheckInterval = 1 // Check every file
|
||||||
|
bm.maxMemoryUsage = 1 // Very low limit to guarantee trigger
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Should detect memory over limit
|
||||||
|
shouldApply := bm.ShouldApplyBackpressure(ctx)
|
||||||
|
assert.True(t, shouldApply)
|
||||||
|
stats := bm.GetStats()
|
||||||
|
assert.True(t, stats.MemoryWarningActive)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("resets warning when memory normalizes", func(t *testing.T) {
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
bm.enabled = true
|
||||||
|
bm.memoryCheckInterval = 1
|
||||||
|
// Simulate warning by first triggering high memory usage
|
||||||
|
bm.maxMemoryUsage = 1 // Very low to trigger warning
|
||||||
|
ctx := context.Background()
|
||||||
|
_ = bm.ShouldApplyBackpressure(ctx)
|
||||||
|
stats := bm.GetStats()
|
||||||
|
assert.True(t, stats.MemoryWarningActive)
|
||||||
|
|
||||||
|
// Now set high limit so we're under it
|
||||||
|
bm.maxMemoryUsage = 1024 * 1024 * 1024 * 10 // 10GB
|
||||||
|
|
||||||
|
shouldApply := bm.ShouldApplyBackpressure(ctx)
|
||||||
|
assert.False(t, shouldApply)
|
||||||
|
|
||||||
|
// Warning should be reset (via public API)
|
||||||
|
stats = bm.GetStats()
|
||||||
|
assert.False(t, stats.MemoryWarningActive)
|
||||||
|
})
|
||||||
|
}
|
||||||
262
fileproc/backpressure_channels_test.go
Normal file
262
fileproc/backpressure_channels_test.go
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
package fileproc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// CI-safe timeout constants
|
||||||
|
fastOpTimeout = 100 * time.Millisecond // Operations that should complete quickly
|
||||||
|
slowOpMinTime = 10 * time.Millisecond // Minimum time for blocking operations
|
||||||
|
)
|
||||||
|
|
||||||
|
// cleanupViperConfig is a test helper that captures and restores viper configuration.
|
||||||
|
// It takes a testing.T and a list of config keys to save/restore.
|
||||||
|
// Returns a cleanup function that should be called via t.Cleanup.
|
||||||
|
func cleanupViperConfig(t *testing.T, keys ...string) {
|
||||||
|
t.Helper()
|
||||||
|
// Capture original values
|
||||||
|
origValues := make(map[string]interface{})
|
||||||
|
for _, key := range keys {
|
||||||
|
origValues[key] = viper.Get(key)
|
||||||
|
}
|
||||||
|
// Register cleanup to restore values
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for key, val := range origValues {
|
||||||
|
if val != nil {
|
||||||
|
viper.Set(key, val)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerCreateChannels(t *testing.T) {
|
||||||
|
t.Run("creates buffered channels when enabled", func(t *testing.T) {
|
||||||
|
// Capture and restore viper config
|
||||||
|
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles, testBackpressureMaxWrites)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxFiles, 10)
|
||||||
|
viper.Set(testBackpressureMaxWrites, 10)
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
fileCh, writeCh := bm.CreateChannels()
|
||||||
|
assert.NotNil(t, fileCh)
|
||||||
|
assert.NotNil(t, writeCh)
|
||||||
|
|
||||||
|
// Test that channels have buffer capacity
|
||||||
|
assert.Greater(t, cap(fileCh), 0)
|
||||||
|
assert.Greater(t, cap(writeCh), 0)
|
||||||
|
|
||||||
|
// Test sending and receiving
|
||||||
|
fileCh <- "test.go"
|
||||||
|
val := <-fileCh
|
||||||
|
assert.Equal(t, "test.go", val)
|
||||||
|
|
||||||
|
writeCh <- WriteRequest{Content: "test content"}
|
||||||
|
writeReq := <-writeCh
|
||||||
|
assert.Equal(t, "test content", writeReq.Content)
|
||||||
|
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("creates unbuffered channels when disabled", func(t *testing.T) {
|
||||||
|
// Use viper to configure instead of direct field access
|
||||||
|
cleanupViperConfig(t, testBackpressureEnabled)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, false)
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
fileCh, writeCh := bm.CreateChannels()
|
||||||
|
assert.NotNil(t, fileCh)
|
||||||
|
assert.NotNil(t, writeCh)
|
||||||
|
|
||||||
|
// Unbuffered channels have capacity 0
|
||||||
|
assert.Equal(t, 0, cap(fileCh))
|
||||||
|
assert.Equal(t, 0, cap(writeCh))
|
||||||
|
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerWaitForChannelSpace(t *testing.T) {
|
||||||
|
t.Run("does nothing when disabled", func(t *testing.T) {
|
||||||
|
// Use viper to configure instead of direct field access
|
||||||
|
cleanupViperConfig(t, testBackpressureEnabled)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, false)
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
fileCh := make(chan string, 1)
|
||||||
|
writeCh := make(chan WriteRequest, 1)
|
||||||
|
|
||||||
|
// Use context with timeout instead of measuring elapsed time
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), fastOpTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Should return immediately (before timeout)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success - operation completed quickly
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("WaitForChannelSpace should return immediately when disabled")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("waits when file channel is nearly full", func(t *testing.T) {
|
||||||
|
// Use viper to configure instead of direct field access
|
||||||
|
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxFiles, 10)
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
// Create channel with exact capacity
|
||||||
|
fileCh := make(chan string, 10)
|
||||||
|
writeCh := make(chan WriteRequest, 10)
|
||||||
|
|
||||||
|
// Fill file channel to >90% (with minimum of 1)
|
||||||
|
target := max(1, int(float64(cap(fileCh))*0.9))
|
||||||
|
for i := 0; i < target; i++ {
|
||||||
|
fileCh <- "file.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test that it blocks by verifying it doesn't complete immediately
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan struct{})
|
||||||
|
start := time.Now()
|
||||||
|
go func() {
|
||||||
|
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Verify it doesn't complete immediately (within first millisecond)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatal("WaitForChannelSpace should block when channel is nearly full")
|
||||||
|
case <-time.After(1 * time.Millisecond):
|
||||||
|
// Good - it's blocking as expected
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for it to complete
|
||||||
|
<-done
|
||||||
|
duration := time.Since(start)
|
||||||
|
// Just verify it took some measurable time (very lenient for CI)
|
||||||
|
assert.GreaterOrEqual(t, duration, 1*time.Millisecond)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
for i := 0; i < target; i++ {
|
||||||
|
<-fileCh
|
||||||
|
}
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("waits when write channel is nearly full", func(t *testing.T) {
|
||||||
|
// Use viper to configure instead of direct field access
|
||||||
|
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxWrites)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxWrites, 10)
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
fileCh := make(chan string, 10)
|
||||||
|
writeCh := make(chan WriteRequest, 10)
|
||||||
|
|
||||||
|
// Fill write channel to >90% (with minimum of 1)
|
||||||
|
target := max(1, int(float64(cap(writeCh))*0.9))
|
||||||
|
for i := 0; i < target; i++ {
|
||||||
|
writeCh <- WriteRequest{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test that it blocks by verifying it doesn't complete immediately
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan struct{})
|
||||||
|
start := time.Now()
|
||||||
|
go func() {
|
||||||
|
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Verify it doesn't complete immediately (within first millisecond)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatal("WaitForChannelSpace should block when channel is nearly full")
|
||||||
|
case <-time.After(1 * time.Millisecond):
|
||||||
|
// Good - it's blocking as expected
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for it to complete
|
||||||
|
<-done
|
||||||
|
duration := time.Since(start)
|
||||||
|
// Just verify it took some measurable time (very lenient for CI)
|
||||||
|
assert.GreaterOrEqual(t, duration, 1*time.Millisecond)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
for i := 0; i < target; i++ {
|
||||||
|
<-writeCh
|
||||||
|
}
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("respects context cancellation", func(t *testing.T) {
|
||||||
|
// Use viper to configure instead of direct field access
|
||||||
|
cleanupViperConfig(t, testBackpressureEnabled, testBackpressureMaxFiles)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxFiles, 10)
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
fileCh := make(chan string, 10)
|
||||||
|
writeCh := make(chan WriteRequest, 10)
|
||||||
|
|
||||||
|
// Fill channel
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
fileCh <- "file.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // Cancel immediately
|
||||||
|
|
||||||
|
// Use timeout to verify it returns quickly
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Should return quickly when context is cancelled
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success - returned due to cancellation
|
||||||
|
case <-time.After(fastOpTimeout):
|
||||||
|
t.Fatal("WaitForChannelSpace should return immediately when context is cancelled")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
<-fileCh
|
||||||
|
}
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
})
|
||||||
|
}
|
||||||
195
fileproc/backpressure_concurrency_test.go
Normal file
195
fileproc/backpressure_concurrency_test.go
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
package fileproc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBackpressureManagerConcurrency(t *testing.T) {
|
||||||
|
// Configure via viper instead of direct field access
|
||||||
|
origEnabled := viper.Get(testBackpressureEnabled)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if origEnabled != nil {
|
||||||
|
viper.Set(testBackpressureEnabled, origEnabled)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
// Multiple goroutines checking backpressure
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
bm.ShouldApplyBackpressure(ctx)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Multiple goroutines applying backpressure
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
bm.ApplyBackpressure(ctx)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Multiple goroutines getting stats
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
bm.GetStats()
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Multiple goroutines creating channels
|
||||||
|
// Note: CreateChannels returns new channels each time, caller owns them
|
||||||
|
type channelResult struct {
|
||||||
|
fileCh chan string
|
||||||
|
writeCh chan WriteRequest
|
||||||
|
}
|
||||||
|
results := make(chan channelResult, 3)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
fileCh, writeCh := bm.CreateChannels()
|
||||||
|
results <- channelResult{fileCh, writeCh}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
close(results)
|
||||||
|
|
||||||
|
// Verify channels are created and have expected properties
|
||||||
|
for result := range results {
|
||||||
|
assert.NotNil(t, result.fileCh)
|
||||||
|
assert.NotNil(t, result.writeCh)
|
||||||
|
// Close channels to prevent resource leak (caller owns them)
|
||||||
|
close(result.fileCh)
|
||||||
|
close(result.writeCh)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify stats are consistent
|
||||||
|
stats := bm.GetStats()
|
||||||
|
assert.GreaterOrEqual(t, stats.FilesProcessed, int64(10))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerIntegration(t *testing.T) {
|
||||||
|
// Configure via viper instead of direct field access
|
||||||
|
origEnabled := viper.Get(testBackpressureEnabled)
|
||||||
|
origMaxFiles := viper.Get(testBackpressureMaxFiles)
|
||||||
|
origMaxWrites := viper.Get(testBackpressureMaxWrites)
|
||||||
|
origCheckInterval := viper.Get(testBackpressureMemoryCheck)
|
||||||
|
origMaxMemory := viper.Get(testBackpressureMaxMemory)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if origEnabled != nil {
|
||||||
|
viper.Set(testBackpressureEnabled, origEnabled)
|
||||||
|
}
|
||||||
|
if origMaxFiles != nil {
|
||||||
|
viper.Set(testBackpressureMaxFiles, origMaxFiles)
|
||||||
|
}
|
||||||
|
if origMaxWrites != nil {
|
||||||
|
viper.Set(testBackpressureMaxWrites, origMaxWrites)
|
||||||
|
}
|
||||||
|
if origCheckInterval != nil {
|
||||||
|
viper.Set(testBackpressureMemoryCheck, origCheckInterval)
|
||||||
|
}
|
||||||
|
if origMaxMemory != nil {
|
||||||
|
viper.Set(testBackpressureMaxMemory, origMaxMemory)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxFiles, 10)
|
||||||
|
viper.Set(testBackpressureMaxWrites, 10)
|
||||||
|
viper.Set(testBackpressureMemoryCheck, 10)
|
||||||
|
viper.Set(testBackpressureMaxMemory, 100*1024*1024) // 100MB
|
||||||
|
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Create channels - caller owns these channels and is responsible for closing them
|
||||||
|
fileCh, writeCh := bm.CreateChannels()
|
||||||
|
require.NotNil(t, fileCh)
|
||||||
|
require.NotNil(t, writeCh)
|
||||||
|
require.Greater(t, cap(fileCh), 0, "fileCh should be buffered")
|
||||||
|
require.Greater(t, cap(writeCh), 0, "writeCh should be buffered")
|
||||||
|
|
||||||
|
// Simulate file processing
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
// Producer
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
// Check for backpressure
|
||||||
|
if bm.ShouldApplyBackpressure(ctx) {
|
||||||
|
bm.ApplyBackpressure(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for channel space if needed
|
||||||
|
bm.WaitForChannelSpace(ctx, fileCh, writeCh)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case fileCh <- "file.txt":
|
||||||
|
// File sent
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Consumer
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
select {
|
||||||
|
case <-fileCh:
|
||||||
|
// Process file (do not manually increment filesProcessed)
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for completion
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Integration test timeout")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log final info
|
||||||
|
bm.LogBackpressureInfo()
|
||||||
|
|
||||||
|
// Check final stats
|
||||||
|
stats := bm.GetStats()
|
||||||
|
assert.GreaterOrEqual(t, stats.FilesProcessed, int64(100))
|
||||||
|
|
||||||
|
// Clean up - caller owns the channels, safe to close now that goroutines have finished
|
||||||
|
close(fileCh)
|
||||||
|
close(writeCh)
|
||||||
|
}
|
||||||
151
fileproc/backpressure_init_test.go
Normal file
151
fileproc/backpressure_init_test.go
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
package fileproc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
// setupViperCleanup is a test helper that captures and restores viper configuration.
|
||||||
|
// It takes a testing.T and a list of config keys to save/restore.
|
||||||
|
func setupViperCleanup(t *testing.T, keys []string) {
|
||||||
|
t.Helper()
|
||||||
|
// Capture original values and track which keys existed
|
||||||
|
origValues := make(map[string]interface{})
|
||||||
|
keysExisted := make(map[string]bool)
|
||||||
|
for _, key := range keys {
|
||||||
|
val := viper.Get(key)
|
||||||
|
origValues[key] = val
|
||||||
|
keysExisted[key] = viper.IsSet(key)
|
||||||
|
}
|
||||||
|
// Register cleanup to restore values
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, key := range keys {
|
||||||
|
if keysExisted[key] {
|
||||||
|
viper.Set(key, origValues[key])
|
||||||
|
} else {
|
||||||
|
// Key didn't exist originally, so remove it
|
||||||
|
allSettings := viper.AllSettings()
|
||||||
|
delete(allSettings, key)
|
||||||
|
viper.Reset()
|
||||||
|
for k, v := range allSettings {
|
||||||
|
viper.Set(k, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewBackpressureManager(t *testing.T) {
|
||||||
|
keys := []string{
|
||||||
|
testBackpressureEnabled,
|
||||||
|
testBackpressureMaxMemory,
|
||||||
|
testBackpressureMemoryCheck,
|
||||||
|
testBackpressureMaxFiles,
|
||||||
|
testBackpressureMaxWrites,
|
||||||
|
}
|
||||||
|
setupViperCleanup(t, keys)
|
||||||
|
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxMemory, 100)
|
||||||
|
viper.Set(testBackpressureMemoryCheck, 10)
|
||||||
|
viper.Set(testBackpressureMaxFiles, 10)
|
||||||
|
viper.Set(testBackpressureMaxWrites, 10)
|
||||||
|
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
assert.NotNil(t, bm)
|
||||||
|
assert.True(t, bm.enabled)
|
||||||
|
assert.Greater(t, bm.maxMemoryUsage, int64(0))
|
||||||
|
assert.Greater(t, bm.memoryCheckInterval, 0)
|
||||||
|
assert.Greater(t, bm.maxPendingFiles, 0)
|
||||||
|
assert.Greater(t, bm.maxPendingWrites, 0)
|
||||||
|
assert.Equal(t, int64(0), bm.filesProcessed)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureStatsStructure(t *testing.T) {
|
||||||
|
// Behavioral test that exercises BackpressureManager and validates stats
|
||||||
|
keys := []string{
|
||||||
|
testBackpressureEnabled,
|
||||||
|
testBackpressureMaxMemory,
|
||||||
|
testBackpressureMemoryCheck,
|
||||||
|
testBackpressureMaxFiles,
|
||||||
|
testBackpressureMaxWrites,
|
||||||
|
}
|
||||||
|
setupViperCleanup(t, keys)
|
||||||
|
|
||||||
|
// Configure backpressure with realistic settings
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMaxMemory, 100*1024*1024) // 100MB
|
||||||
|
viper.Set(testBackpressureMemoryCheck, 1) // Check every file
|
||||||
|
viper.Set(testBackpressureMaxFiles, 1000)
|
||||||
|
viper.Set(testBackpressureMaxWrites, 500)
|
||||||
|
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Simulate processing files
|
||||||
|
initialStats := bm.GetStats()
|
||||||
|
assert.True(t, initialStats.Enabled, "backpressure should be enabled")
|
||||||
|
assert.Equal(t, int64(0), initialStats.FilesProcessed, "initially no files processed")
|
||||||
|
|
||||||
|
// Capture initial timestamp to verify it gets updated
|
||||||
|
initialLastCheck := initialStats.LastMemoryCheck
|
||||||
|
|
||||||
|
// Process some files to trigger memory checks
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
bm.ShouldApplyBackpressure(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify stats reflect the operations
|
||||||
|
stats := bm.GetStats()
|
||||||
|
assert.True(t, stats.Enabled, "enabled flag should be set")
|
||||||
|
assert.Equal(t, int64(5), stats.FilesProcessed, "should have processed 5 files")
|
||||||
|
assert.Greater(t, stats.CurrentMemoryUsage, int64(0), "memory usage should be tracked")
|
||||||
|
assert.Equal(t, int64(100*1024*1024), stats.MaxMemoryUsage, "max memory should match config")
|
||||||
|
assert.Equal(t, 1000, stats.MaxPendingFiles, "maxPendingFiles should match config")
|
||||||
|
assert.Equal(t, 500, stats.MaxPendingWrites, "maxPendingWrites should match config")
|
||||||
|
assert.True(t, stats.LastMemoryCheck.After(initialLastCheck) || stats.LastMemoryCheck.Equal(initialLastCheck),
|
||||||
|
"lastMemoryCheck should be updated or remain initialized")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackpressureManagerGetStats(t *testing.T) {
|
||||||
|
keys := []string{
|
||||||
|
testBackpressureEnabled,
|
||||||
|
testBackpressureMemoryCheck,
|
||||||
|
}
|
||||||
|
setupViperCleanup(t, keys)
|
||||||
|
|
||||||
|
// Ensure config enables backpressure and checks every call
|
||||||
|
viper.Set(testBackpressureEnabled, true)
|
||||||
|
viper.Set(testBackpressureMemoryCheck, 1)
|
||||||
|
|
||||||
|
bm := NewBackpressureManager()
|
||||||
|
|
||||||
|
// Capture initial timestamp to verify it gets updated
|
||||||
|
initialStats := bm.GetStats()
|
||||||
|
initialLastCheck := initialStats.LastMemoryCheck
|
||||||
|
|
||||||
|
// Process some files to update stats
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
bm.ShouldApplyBackpressure(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := bm.GetStats()
|
||||||
|
|
||||||
|
assert.True(t, stats.Enabled)
|
||||||
|
assert.Equal(t, int64(5), stats.FilesProcessed)
|
||||||
|
assert.Greater(t, stats.CurrentMemoryUsage, int64(0))
|
||||||
|
assert.Equal(t, bm.maxMemoryUsage, stats.MaxMemoryUsage)
|
||||||
|
assert.Equal(t, bm.maxPendingFiles, stats.MaxPendingFiles)
|
||||||
|
assert.Equal(t, bm.maxPendingWrites, stats.MaxPendingWrites)
|
||||||
|
|
||||||
|
// LastMemoryCheck should be updated after processing files (memoryCheckInterval=1)
|
||||||
|
assert.True(t, stats.LastMemoryCheck.After(initialLastCheck),
|
||||||
|
"lastMemoryCheck should be updated after memory checks")
|
||||||
|
}
|
||||||
@@ -1,9 +1,162 @@
|
|||||||
package fileproc
|
package fileproc
|
||||||
|
|
||||||
import "strings"
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// MaxRegistryEntries is the maximum number of entries allowed in registry config slices/maps.
|
||||||
|
MaxRegistryEntries = 1000
|
||||||
|
// MaxExtensionLength is the maximum length for a single extension string.
|
||||||
|
MaxExtensionLength = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
// RegistryConfig holds configuration for file type registry.
|
||||||
|
// All paths must be relative without path traversal (no ".." or leading "/").
|
||||||
|
// Extensions in CustomLanguages keys must start with "." or be alphanumeric with underscore/hyphen.
|
||||||
|
type RegistryConfig struct {
|
||||||
|
// CustomImages: file extensions to treat as images (e.g., ".svg", ".webp").
|
||||||
|
// Must be relative paths without ".." or leading separators.
|
||||||
|
CustomImages []string
|
||||||
|
|
||||||
|
// CustomBinary: file extensions to treat as binary (e.g., ".bin", ".dat").
|
||||||
|
// Must be relative paths without ".." or leading separators.
|
||||||
|
CustomBinary []string
|
||||||
|
|
||||||
|
// CustomLanguages: maps file extensions to language names (e.g., {".tsx": "TypeScript"}).
|
||||||
|
// Keys must start with "." or be alphanumeric with underscore/hyphen.
|
||||||
|
CustomLanguages map[string]string
|
||||||
|
|
||||||
|
// DisabledImages: image extensions to disable from default registry.
|
||||||
|
DisabledImages []string
|
||||||
|
|
||||||
|
// DisabledBinary: binary extensions to disable from default registry.
|
||||||
|
DisabledBinary []string
|
||||||
|
|
||||||
|
// DisabledLanguages: language extensions to disable from default registry.
|
||||||
|
DisabledLanguages []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks the RegistryConfig for invalid entries and enforces limits.
|
||||||
|
func (c *RegistryConfig) Validate() error {
|
||||||
|
// Validate CustomImages
|
||||||
|
if err := validateExtensionSlice(c.CustomImages, "CustomImages"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate CustomBinary
|
||||||
|
if err := validateExtensionSlice(c.CustomBinary, "CustomBinary"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate CustomLanguages
|
||||||
|
if len(c.CustomLanguages) > MaxRegistryEntries {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"CustomLanguages exceeds maximum entries (%d > %d)",
|
||||||
|
len(c.CustomLanguages),
|
||||||
|
MaxRegistryEntries,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for ext, lang := range c.CustomLanguages {
|
||||||
|
if err := validateExtension(ext, "CustomLanguages key"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(lang) > MaxExtensionLength {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"CustomLanguages value %q exceeds maximum length (%d > %d)",
|
||||||
|
lang,
|
||||||
|
len(lang),
|
||||||
|
MaxExtensionLength,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate Disabled slices
|
||||||
|
if err := validateExtensionSlice(c.DisabledImages, "DisabledImages"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateExtensionSlice(c.DisabledBinary, "DisabledBinary"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return validateExtensionSlice(c.DisabledLanguages, "DisabledLanguages")
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateExtensionSlice validates a slice of extensions for path safety and limits.
|
||||||
|
func validateExtensionSlice(slice []string, fieldName string) error {
|
||||||
|
if len(slice) > MaxRegistryEntries {
|
||||||
|
return fmt.Errorf("%s exceeds maximum entries (%d > %d)", fieldName, len(slice), MaxRegistryEntries)
|
||||||
|
}
|
||||||
|
for _, ext := range slice {
|
||||||
|
if err := validateExtension(ext, fieldName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateExtension validates a single extension for path safety.
|
||||||
|
//
|
||||||
|
//revive:disable-next-line:cyclomatic
|
||||||
|
func validateExtension(ext, context string) error {
|
||||||
|
// Reject empty strings
|
||||||
|
if ext == "" {
|
||||||
|
return fmt.Errorf("%s entry cannot be empty", context)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ext) > MaxExtensionLength {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s entry %q exceeds maximum length (%d > %d)",
|
||||||
|
context, ext, len(ext), MaxExtensionLength,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reject absolute paths
|
||||||
|
if filepath.IsAbs(ext) {
|
||||||
|
return fmt.Errorf("%s entry %q is an absolute path (not allowed)", context, ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reject path traversal
|
||||||
|
if strings.Contains(ext, "..") {
|
||||||
|
return fmt.Errorf("%s entry %q contains path traversal (not allowed)", context, ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
// For extensions, verify they start with "." or are alphanumeric
|
||||||
|
if strings.HasPrefix(ext, ".") {
|
||||||
|
// Reject extensions containing path separators
|
||||||
|
if strings.ContainsRune(ext, filepath.Separator) || strings.ContainsRune(ext, '/') ||
|
||||||
|
strings.ContainsRune(ext, '\\') {
|
||||||
|
return fmt.Errorf("%s entry %q contains path separators (not allowed)", context, ext)
|
||||||
|
}
|
||||||
|
// Valid extension format
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if purely alphanumeric (for bare names)
|
||||||
|
for _, r := range ext {
|
||||||
|
isValid := (r >= 'a' && r <= 'z') ||
|
||||||
|
(r >= 'A' && r <= 'Z') ||
|
||||||
|
(r >= '0' && r <= '9') ||
|
||||||
|
r == '_' || r == '-'
|
||||||
|
if !isValid {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s entry %q contains invalid characters (must start with '.' or be alphanumeric/_/-)",
|
||||||
|
context,
|
||||||
|
ext,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ApplyCustomExtensions applies custom extensions from configuration.
|
// ApplyCustomExtensions applies custom extensions from configuration.
|
||||||
func (r *FileTypeRegistry) ApplyCustomExtensions(customImages, customBinary []string, customLanguages map[string]string) {
|
func (r *FileTypeRegistry) ApplyCustomExtensions(
|
||||||
|
customImages, customBinary []string,
|
||||||
|
customLanguages map[string]string,
|
||||||
|
) {
|
||||||
// Add custom image extensions
|
// Add custom image extensions
|
||||||
r.addExtensions(customImages, r.AddImageExtension)
|
r.addExtensions(customImages, r.AddImageExtension)
|
||||||
|
|
||||||
@@ -29,12 +182,24 @@ func (r *FileTypeRegistry) addExtensions(extensions []string, adder func(string)
|
|||||||
|
|
||||||
// ConfigureFromSettings applies configuration settings to the registry.
|
// ConfigureFromSettings applies configuration settings to the registry.
|
||||||
// This function is called from main.go after config is loaded to avoid circular imports.
|
// This function is called from main.go after config is loaded to avoid circular imports.
|
||||||
func ConfigureFromSettings(
|
// It validates the configuration before applying it.
|
||||||
customImages, customBinary []string,
|
func ConfigureFromSettings(config RegistryConfig) error {
|
||||||
customLanguages map[string]string,
|
// Validate configuration first
|
||||||
disabledImages, disabledBinary, disabledLanguages []string,
|
if err := config.Validate(); err != nil {
|
||||||
) {
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
registry := GetDefaultRegistry()
|
registry := GetDefaultRegistry()
|
||||||
registry.ApplyCustomExtensions(customImages, customBinary, customLanguages)
|
|
||||||
registry.DisableExtensions(disabledImages, disabledBinary, disabledLanguages)
|
// Only apply custom extensions if they are non-empty (len() for nil slices/maps is zero)
|
||||||
|
if len(config.CustomImages) > 0 || len(config.CustomBinary) > 0 || len(config.CustomLanguages) > 0 {
|
||||||
|
registry.ApplyCustomExtensions(config.CustomImages, config.CustomBinary, config.CustomLanguages)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only disable extensions if they are non-empty
|
||||||
|
if len(config.DisabledImages) > 0 || len(config.DisabledBinary) > 0 || len(config.DisabledLanguages) > 0 {
|
||||||
|
registry.DisableExtensions(config.DisabledImages, config.DisabledBinary, config.DisabledLanguages)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,10 +14,10 @@ func TestFileTypeRegistry_ThreadSafety(t *testing.T) {
|
|||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
// Test concurrent read operations
|
// Test concurrent read operations
|
||||||
t.Run("ConcurrentReads", func(t *testing.T) {
|
t.Run("ConcurrentReads", func(_ *testing.T) {
|
||||||
for i := 0; i < numGoroutines; i++ {
|
for i := 0; i < numGoroutines; i++ {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func(id int) {
|
go func(_ int) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
registry := GetDefaultRegistry()
|
registry := GetDefaultRegistry()
|
||||||
|
|
||||||
@@ -102,4 +102,4 @@ func TestFileTypeRegistry_ThreadSafety(t *testing.T) {
|
|||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
package fileproc
|
package fileproc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestFileTypeRegistry_Configuration tests the configuration functionality.
|
// TestFileTypeRegistry_Configuration tests the configuration functionality.
|
||||||
@@ -142,7 +143,7 @@ func TestFileTypeRegistry_Configuration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// Test case insensitive handling
|
// Test case-insensitive handling
|
||||||
t.Run("CaseInsensitiveHandling", func(t *testing.T) {
|
t.Run("CaseInsensitiveHandling", func(t *testing.T) {
|
||||||
registry := &FileTypeRegistry{
|
registry := &FileTypeRegistry{
|
||||||
imageExts: make(map[string]bool),
|
imageExts: make(map[string]bool),
|
||||||
@@ -184,8 +185,9 @@ func TestFileTypeRegistry_Configuration(t *testing.T) {
|
|||||||
// TestConfigureFromSettings tests the global configuration function.
|
// TestConfigureFromSettings tests the global configuration function.
|
||||||
func TestConfigureFromSettings(t *testing.T) {
|
func TestConfigureFromSettings(t *testing.T) {
|
||||||
// Reset registry to ensure clean state
|
// Reset registry to ensure clean state
|
||||||
registryOnce = sync.Once{}
|
ResetRegistryForTesting()
|
||||||
registry = nil
|
// Ensure cleanup runs even if test fails
|
||||||
|
t.Cleanup(ResetRegistryForTesting)
|
||||||
|
|
||||||
// Test configuration application
|
// Test configuration application
|
||||||
customImages := []string{".webp", ".avif"}
|
customImages := []string{".webp", ".avif"}
|
||||||
@@ -195,14 +197,15 @@ func TestConfigureFromSettings(t *testing.T) {
|
|||||||
disabledBinary := []string{".exe"} // Disable default extension
|
disabledBinary := []string{".exe"} // Disable default extension
|
||||||
disabledLanguages := []string{".rb"} // Disable default extension
|
disabledLanguages := []string{".rb"} // Disable default extension
|
||||||
|
|
||||||
ConfigureFromSettings(
|
err := ConfigureFromSettings(RegistryConfig{
|
||||||
customImages,
|
CustomImages: customImages,
|
||||||
customBinary,
|
CustomBinary: customBinary,
|
||||||
customLanguages,
|
CustomLanguages: customLanguages,
|
||||||
disabledImages,
|
DisabledImages: disabledImages,
|
||||||
disabledBinary,
|
DisabledBinary: disabledBinary,
|
||||||
disabledLanguages,
|
DisabledLanguages: disabledLanguages,
|
||||||
)
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Test that custom extensions work
|
// Test that custom extensions work
|
||||||
if !IsImage("test.webp") {
|
if !IsImage("test.webp") {
|
||||||
@@ -238,14 +241,15 @@ func TestConfigureFromSettings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Test multiple calls don't override previous configuration
|
// Test multiple calls don't override previous configuration
|
||||||
ConfigureFromSettings(
|
err = ConfigureFromSettings(RegistryConfig{
|
||||||
[]string{".extra"},
|
CustomImages: []string{".extra"},
|
||||||
[]string{},
|
CustomBinary: []string{},
|
||||||
map[string]string{},
|
CustomLanguages: map[string]string{},
|
||||||
[]string{},
|
DisabledImages: []string{},
|
||||||
[]string{},
|
DisabledBinary: []string{},
|
||||||
[]string{},
|
DisabledLanguages: []string{},
|
||||||
)
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Previous configuration should still work
|
// Previous configuration should still work
|
||||||
if !IsImage("test.webp") {
|
if !IsImage("test.webp") {
|
||||||
@@ -255,4 +259,4 @@ func TestConfigureFromSettings(t *testing.T) {
|
|||||||
if !IsImage("test.extra") {
|
if !IsImage("test.extra") {
|
||||||
t.Error("Expected new configuration to be applied")
|
t.Error("Expected new configuration to be applied")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,21 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// newTestRegistry creates a fresh registry instance for testing to avoid global state pollution.
|
||||||
|
func newTestRegistry() *FileTypeRegistry {
|
||||||
|
return &FileTypeRegistry{
|
||||||
|
imageExts: getImageExtensions(),
|
||||||
|
binaryExts: getBinaryExtensions(),
|
||||||
|
languageMap: getLanguageMap(),
|
||||||
|
extCache: make(map[string]string, 1000),
|
||||||
|
resultCache: make(map[string]FileTypeResult, 500),
|
||||||
|
maxCacheSize: 500,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestFileTypeRegistry_LanguageDetection tests the language detection functionality.
|
// TestFileTypeRegistry_LanguageDetection tests the language detection functionality.
|
||||||
func TestFileTypeRegistry_LanguageDetection(t *testing.T) {
|
func TestFileTypeRegistry_LanguageDetection(t *testing.T) {
|
||||||
registry := GetDefaultRegistry()
|
registry := newTestRegistry()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
filename string
|
filename string
|
||||||
@@ -94,7 +106,7 @@ func TestFileTypeRegistry_LanguageDetection(t *testing.T) {
|
|||||||
|
|
||||||
// TestFileTypeRegistry_ImageDetection tests the image detection functionality.
|
// TestFileTypeRegistry_ImageDetection tests the image detection functionality.
|
||||||
func TestFileTypeRegistry_ImageDetection(t *testing.T) {
|
func TestFileTypeRegistry_ImageDetection(t *testing.T) {
|
||||||
registry := GetDefaultRegistry()
|
registry := newTestRegistry()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
filename string
|
filename string
|
||||||
@@ -144,7 +156,7 @@ func TestFileTypeRegistry_ImageDetection(t *testing.T) {
|
|||||||
|
|
||||||
// TestFileTypeRegistry_BinaryDetection tests the binary detection functionality.
|
// TestFileTypeRegistry_BinaryDetection tests the binary detection functionality.
|
||||||
func TestFileTypeRegistry_BinaryDetection(t *testing.T) {
|
func TestFileTypeRegistry_BinaryDetection(t *testing.T) {
|
||||||
registry := GetDefaultRegistry()
|
registry := newTestRegistry()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
filename string
|
filename string
|
||||||
@@ -208,11 +220,11 @@ func TestFileTypeRegistry_BinaryDetection(t *testing.T) {
|
|||||||
{"page.html", false},
|
{"page.html", false},
|
||||||
|
|
||||||
// Edge cases
|
// Edge cases
|
||||||
{"", false}, // Empty filename
|
{"", false}, // Empty filename
|
||||||
{"binary", false}, // No extension
|
{"binary", false}, // No extension
|
||||||
{".exe", true}, // Just extension
|
{".exe", true}, // Just extension
|
||||||
{"file.exe.txt", false}, // Multiple extensions
|
{"file.exe.txt", false}, // Multiple extensions
|
||||||
{"file.unknown", false}, // Unknown extension
|
{"file.unknown", false}, // Unknown extension
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -223,4 +235,4 @@ func TestFileTypeRegistry_BinaryDetection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func TestFileTypeRegistry_EdgeCases(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range edgeCases {
|
for _, tc := range edgeCases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(_ *testing.T) {
|
||||||
// These should not panic
|
// These should not panic
|
||||||
_ = registry.IsImage(tc.filename)
|
_ = registry.IsImage(tc.filename)
|
||||||
_ = registry.IsBinary(tc.filename)
|
_ = registry.IsBinary(tc.filename)
|
||||||
@@ -125,4 +125,4 @@ func BenchmarkFileTypeRegistry_ConcurrentAccess(b *testing.B) {
|
|||||||
_ = GetLanguage(filename)
|
_ = GetLanguage(filename)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func TestFileTypeRegistry_ModificationMethods(t *testing.T) {
|
|||||||
t.Errorf("Expected .webp to be recognized as image after adding")
|
t.Errorf("Expected .webp to be recognized as image after adding")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test case insensitive addition
|
// Test case-insensitive addition
|
||||||
registry.AddImageExtension(".AVIF")
|
registry.AddImageExtension(".AVIF")
|
||||||
if !registry.IsImage("test.avif") {
|
if !registry.IsImage("test.avif") {
|
||||||
t.Errorf("Expected .avif to be recognized as image after adding .AVIF")
|
t.Errorf("Expected .avif to be recognized as image after adding .AVIF")
|
||||||
@@ -51,7 +51,7 @@ func TestFileTypeRegistry_ModificationMethods(t *testing.T) {
|
|||||||
t.Errorf("Expected .custom to be recognized as binary after adding")
|
t.Errorf("Expected .custom to be recognized as binary after adding")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test case insensitive addition
|
// Test case-insensitive addition
|
||||||
registry.AddBinaryExtension(".SPECIAL")
|
registry.AddBinaryExtension(".SPECIAL")
|
||||||
if !registry.IsBinary("file.special") {
|
if !registry.IsBinary("file.special") {
|
||||||
t.Errorf("Expected .special to be recognized as binary after adding .SPECIAL")
|
t.Errorf("Expected .special to be recognized as binary after adding .SPECIAL")
|
||||||
@@ -81,7 +81,7 @@ func TestFileTypeRegistry_ModificationMethods(t *testing.T) {
|
|||||||
t.Errorf("Expected CustomLang, got %s", lang)
|
t.Errorf("Expected CustomLang, got %s", lang)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test case insensitive addition
|
// Test case-insensitive addition
|
||||||
registry.AddLanguageMapping(".ABC", "UpperLang")
|
registry.AddLanguageMapping(".ABC", "UpperLang")
|
||||||
if lang := registry.GetLanguage("file.abc"); lang != "UpperLang" {
|
if lang := registry.GetLanguage("file.abc"); lang != "UpperLang" {
|
||||||
t.Errorf("Expected UpperLang, got %s", lang)
|
t.Errorf("Expected UpperLang, got %s", lang)
|
||||||
@@ -134,4 +134,4 @@ func TestFileTypeRegistry_DefaultRegistryConsistency(t *testing.T) {
|
|||||||
t.Errorf("Iteration %d: Expected .txt to not be recognized as binary", i)
|
t.Errorf("Iteration %d: Expected .txt to not be recognized as binary", i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// JSONWriter handles JSON format output with streaming support.
|
// JSONWriter handles JSON format output with streaming support.
|
||||||
@@ -27,27 +27,42 @@ func NewJSONWriter(outFile *os.File) *JSONWriter {
|
|||||||
func (w *JSONWriter) Start(prefix, suffix string) error {
|
func (w *JSONWriter) Start(prefix, suffix string) error {
|
||||||
// Start JSON structure
|
// Start JSON structure
|
||||||
if _, err := w.outFile.WriteString(`{"prefix":"`); err != nil {
|
if _, err := w.outFile.WriteString(`{"prefix":"`); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON start")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON start",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write escaped prefix
|
// Write escaped prefix
|
||||||
escapedPrefix := utils.EscapeForJSON(prefix)
|
escapedPrefix := gibidiutils.EscapeForJSON(prefix)
|
||||||
if err := utils.WriteWithErrorWrap(w.outFile, escapedPrefix, "failed to write JSON prefix", ""); err != nil {
|
if err := gibidiutils.WriteWithErrorWrap(w.outFile, escapedPrefix, "failed to write JSON prefix", ""); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := w.outFile.WriteString(`","suffix":"`); err != nil {
|
if _, err := w.outFile.WriteString(`","suffix":"`); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON middle")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON middle",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write escaped suffix
|
// Write escaped suffix
|
||||||
escapedSuffix := utils.EscapeForJSON(suffix)
|
escapedSuffix := gibidiutils.EscapeForJSON(suffix)
|
||||||
if err := utils.WriteWithErrorWrap(w.outFile, escapedSuffix, "failed to write JSON suffix", ""); err != nil {
|
if err := gibidiutils.WriteWithErrorWrap(w.outFile, escapedSuffix, "failed to write JSON suffix", ""); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := w.outFile.WriteString(`","files":[`); err != nil {
|
if _, err := w.outFile.WriteString(`","files":[`); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON files start")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON files start",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -57,7 +72,12 @@ func (w *JSONWriter) Start(prefix, suffix string) error {
|
|||||||
func (w *JSONWriter) WriteFile(req WriteRequest) error {
|
func (w *JSONWriter) WriteFile(req WriteRequest) error {
|
||||||
if !w.firstFile {
|
if !w.firstFile {
|
||||||
if _, err := w.outFile.WriteString(","); err != nil {
|
if _, err := w.outFile.WriteString(","); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON separator")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON separator",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
w.firstFile = false
|
w.firstFile = false
|
||||||
@@ -72,21 +92,24 @@ func (w *JSONWriter) WriteFile(req WriteRequest) error {
|
|||||||
func (w *JSONWriter) Close() error {
|
func (w *JSONWriter) Close() error {
|
||||||
// Close JSON structure
|
// Close JSON structure
|
||||||
if _, err := w.outFile.WriteString("]}"); err != nil {
|
if _, err := w.outFile.WriteString("]}"); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON end")
|
return gibidiutils.WrapError(err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite, "failed to write JSON end")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeStreaming writes a large file as JSON in streaming chunks.
|
// writeStreaming writes a large file as JSON in streaming chunks.
|
||||||
func (w *JSONWriter) writeStreaming(req WriteRequest) error {
|
func (w *JSONWriter) writeStreaming(req WriteRequest) error {
|
||||||
defer utils.SafeCloseReader(req.Reader, req.Path)
|
defer gibidiutils.SafeCloseReader(req.Reader, req.Path)
|
||||||
|
|
||||||
language := detectLanguage(req.Path)
|
language := detectLanguage(req.Path)
|
||||||
|
|
||||||
// Write file start
|
// Write file start
|
||||||
escapedPath := utils.EscapeForJSON(req.Path)
|
escapedPath := gibidiutils.EscapeForJSON(req.Path)
|
||||||
if _, err := fmt.Fprintf(w.outFile, `{"path":"%s","language":"%s","content":"`, escapedPath, language); err != nil {
|
if _, err := fmt.Fprintf(w.outFile, `{"path":"%s","language":"%s","content":"`, escapedPath, language); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON file start").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON file start",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream content with JSON escaping
|
// Stream content with JSON escaping
|
||||||
@@ -96,7 +119,10 @@ func (w *JSONWriter) writeStreaming(req WriteRequest) error {
|
|||||||
|
|
||||||
// Write file end
|
// Write file end
|
||||||
if _, err := w.outFile.WriteString(`"}`); err != nil {
|
if _, err := w.outFile.WriteString(`"}`); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON file end").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON file end",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -113,25 +139,29 @@ func (w *JSONWriter) writeInline(req WriteRequest) error {
|
|||||||
|
|
||||||
encoded, err := json.Marshal(fileData)
|
encoded, err := json.Marshal(fileData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingEncode, "failed to marshal JSON").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingEncode,
|
||||||
|
"failed to marshal JSON",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := w.outFile.Write(encoded); err != nil {
|
if _, err := w.outFile.Write(encoded); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write JSON file").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write JSON file",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// streamJSONContent streams content with JSON escaping.
|
// streamJSONContent streams content with JSON escaping.
|
||||||
func (w *JSONWriter) streamJSONContent(reader io.Reader, path string) error {
|
func (w *JSONWriter) streamJSONContent(reader io.Reader, path string) error {
|
||||||
return utils.StreamContent(reader, w.outFile, StreamChunkSize, path, func(chunk []byte) []byte {
|
return gibidiutils.StreamContent(reader, w.outFile, StreamChunkSize, path, func(chunk []byte) []byte {
|
||||||
escaped := utils.EscapeForJSON(string(chunk))
|
escaped := gibidiutils.EscapeForJSON(string(chunk))
|
||||||
return []byte(escaped)
|
return []byte(escaped)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// startJSONWriter handles JSON format output with streaming support.
|
// startJSONWriter handles JSON format output with streaming support.
|
||||||
func startJSONWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
|
func startJSONWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
@@ -140,19 +170,19 @@ func startJSONWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<-
|
|||||||
|
|
||||||
// Start writing
|
// Start writing
|
||||||
if err := writer.Start(prefix, suffix); err != nil {
|
if err := writer.Start(prefix, suffix); err != nil {
|
||||||
utils.LogError("Failed to write JSON start", err)
|
gibidiutils.LogError("Failed to write JSON start", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process files
|
// Process files
|
||||||
for req := range writeCh {
|
for req := range writeCh {
|
||||||
if err := writer.WriteFile(req); err != nil {
|
if err := writer.WriteFile(req); err != nil {
|
||||||
utils.LogError("Failed to write JSON file", err)
|
gibidiutils.LogError("Failed to write JSON file", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close writer
|
// Close writer
|
||||||
if err := writer.Close(); err != nil {
|
if err := writer.Close(); err != nil {
|
||||||
utils.LogError("Failed to write JSON end", err)
|
gibidiutils.LogError("Failed to write JSON end", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,11 +4,13 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MarkdownWriter handles markdown format output with streaming support.
|
// MarkdownWriter handles Markdown format output with streaming support.
|
||||||
type MarkdownWriter struct {
|
type MarkdownWriter struct {
|
||||||
outFile *os.File
|
outFile *os.File
|
||||||
}
|
}
|
||||||
@@ -19,16 +21,21 @@ func NewMarkdownWriter(outFile *os.File) *MarkdownWriter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Start writes the markdown header.
|
// Start writes the markdown header.
|
||||||
func (w *MarkdownWriter) Start(prefix, suffix string) error {
|
func (w *MarkdownWriter) Start(prefix, _ string) error {
|
||||||
if prefix != "" {
|
if prefix != "" {
|
||||||
if _, err := fmt.Fprintf(w.outFile, "# %s\n\n", prefix); err != nil {
|
if _, err := fmt.Fprintf(w.outFile, "# %s\n\n", prefix); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write prefix")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write prefix",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteFile writes a file entry in markdown format.
|
// WriteFile writes a file entry in Markdown format.
|
||||||
func (w *MarkdownWriter) WriteFile(req WriteRequest) error {
|
func (w *MarkdownWriter) WriteFile(req WriteRequest) error {
|
||||||
if req.IsStream {
|
if req.IsStream {
|
||||||
return w.writeStreaming(req)
|
return w.writeStreaming(req)
|
||||||
@@ -40,21 +47,99 @@ func (w *MarkdownWriter) WriteFile(req WriteRequest) error {
|
|||||||
func (w *MarkdownWriter) Close(suffix string) error {
|
func (w *MarkdownWriter) Close(suffix string) error {
|
||||||
if suffix != "" {
|
if suffix != "" {
|
||||||
if _, err := fmt.Fprintf(w.outFile, "\n# %s\n", suffix); err != nil {
|
if _, err := fmt.Fprintf(w.outFile, "\n# %s\n", suffix); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write suffix")
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write suffix",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validateMarkdownPath validates a file path for markdown output.
|
||||||
|
func validateMarkdownPath(path string) error {
|
||||||
|
trimmed := strings.TrimSpace(path)
|
||||||
|
if trimmed == "" {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationRequired,
|
||||||
|
"file path cannot be empty",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reject absolute paths
|
||||||
|
if filepath.IsAbs(trimmed) {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"absolute paths are not allowed",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{"path": trimmed},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean and validate path components
|
||||||
|
cleaned := filepath.Clean(trimmed)
|
||||||
|
if filepath.IsAbs(cleaned) || strings.HasPrefix(cleaned, "/") {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path must be relative",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{"path": trimmed, "cleaned": cleaned},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for path traversal in components
|
||||||
|
components := strings.Split(filepath.ToSlash(cleaned), "/")
|
||||||
|
for _, component := range components {
|
||||||
|
if component == ".." {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path traversal not allowed",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{"path": trimmed, "cleaned": cleaned},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// writeStreaming writes a large file in streaming chunks.
|
// writeStreaming writes a large file in streaming chunks.
|
||||||
func (w *MarkdownWriter) writeStreaming(req WriteRequest) error {
|
func (w *MarkdownWriter) writeStreaming(req WriteRequest) error {
|
||||||
defer w.closeReader(req.Reader, req.Path)
|
// Validate path before use
|
||||||
|
if err := validateMarkdownPath(req.Path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for nil reader
|
||||||
|
if req.Reader == nil {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationRequired,
|
||||||
|
"nil reader in write request",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
).WithFilePath(req.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer gibidiutils.SafeCloseReader(req.Reader, req.Path)
|
||||||
|
|
||||||
language := detectLanguage(req.Path)
|
language := detectLanguage(req.Path)
|
||||||
|
|
||||||
// Write file header
|
// Write file header
|
||||||
if _, err := fmt.Fprintf(w.outFile, "## File: `%s`\n```%s\n", req.Path, language); err != nil {
|
safePath := gibidiutils.EscapeForMarkdown(req.Path)
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write file header").WithFilePath(req.Path)
|
if _, err := fmt.Fprintf(w.outFile, "## File: `%s`\n```%s\n", safePath, language); err != nil {
|
||||||
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write file header",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream file content in chunks
|
// Stream file content in chunks
|
||||||
@@ -64,7 +149,10 @@ func (w *MarkdownWriter) writeStreaming(req WriteRequest) error {
|
|||||||
|
|
||||||
// Write file footer
|
// Write file footer
|
||||||
if _, err := w.outFile.WriteString("\n```\n\n"); err != nil {
|
if _, err := w.outFile.WriteString("\n```\n\n"); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write file footer").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write file footer",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -72,68 +160,55 @@ func (w *MarkdownWriter) writeStreaming(req WriteRequest) error {
|
|||||||
|
|
||||||
// writeInline writes a small file directly from content.
|
// writeInline writes a small file directly from content.
|
||||||
func (w *MarkdownWriter) writeInline(req WriteRequest) error {
|
func (w *MarkdownWriter) writeInline(req WriteRequest) error {
|
||||||
|
// Validate path before use
|
||||||
|
if err := validateMarkdownPath(req.Path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
language := detectLanguage(req.Path)
|
language := detectLanguage(req.Path)
|
||||||
formatted := fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", req.Path, language, req.Content)
|
safePath := gibidiutils.EscapeForMarkdown(req.Path)
|
||||||
|
formatted := fmt.Sprintf("## File: `%s`\n```%s\n%s\n```\n\n", safePath, language, req.Content)
|
||||||
|
|
||||||
if _, err := w.outFile.WriteString(formatted); err != nil {
|
if _, err := w.outFile.WriteString(formatted); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write inline content").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write inline content",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// streamContent streams file content in chunks.
|
// streamContent streams file content in chunks.
|
||||||
func (w *MarkdownWriter) streamContent(reader io.Reader, path string) error {
|
func (w *MarkdownWriter) streamContent(reader io.Reader, path string) error {
|
||||||
buf := make([]byte, StreamChunkSize)
|
return gibidiutils.StreamContent(reader, w.outFile, StreamChunkSize, path, nil)
|
||||||
for {
|
|
||||||
n, err := reader.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
if _, writeErr := w.outFile.Write(buf[:n]); writeErr != nil {
|
|
||||||
return utils.WrapError(writeErr, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write chunk").WithFilePath(path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIORead, "failed to read chunk").WithFilePath(path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// closeReader safely closes a reader if it implements io.Closer.
|
// startMarkdownWriter handles Markdown format output with streaming support.
|
||||||
func (w *MarkdownWriter) closeReader(reader io.Reader, path string) {
|
func startMarkdownWriter(
|
||||||
if closer, ok := reader.(io.Closer); ok {
|
outFile *os.File,
|
||||||
if err := closer.Close(); err != nil {
|
writeCh <-chan WriteRequest,
|
||||||
utils.LogError(
|
done chan<- struct{},
|
||||||
"Failed to close file reader",
|
prefix, suffix string,
|
||||||
utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOClose, "failed to close file reader").WithFilePath(path),
|
) {
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startMarkdownWriter handles markdown format output with streaming support.
|
|
||||||
func startMarkdownWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
|
|
||||||
defer close(done)
|
defer close(done)
|
||||||
|
|
||||||
writer := NewMarkdownWriter(outFile)
|
writer := NewMarkdownWriter(outFile)
|
||||||
|
|
||||||
// Start writing
|
// Start writing
|
||||||
if err := writer.Start(prefix, suffix); err != nil {
|
if err := writer.Start(prefix, suffix); err != nil {
|
||||||
utils.LogError("Failed to write markdown prefix", err)
|
gibidiutils.LogError("Failed to write markdown prefix", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process files
|
// Process files
|
||||||
for req := range writeCh {
|
for req := range writeCh {
|
||||||
if err := writer.WriteFile(req); err != nil {
|
if err := writer.WriteFile(req); err != nil {
|
||||||
utils.LogError("Failed to write markdown file", err)
|
gibidiutils.LogError("Failed to write markdown file", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close writer
|
// Close writer
|
||||||
if err := writer.Close(suffix); err != nil {
|
if err := writer.Close(suffix); err != nil {
|
||||||
utils.LogError("Failed to write markdown suffix", err)
|
gibidiutils.LogError("Failed to write markdown suffix", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package fileproc
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
@@ -13,7 +14,7 @@ import (
|
|||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/config"
|
"github.com/ivuorinen/gibidify/config"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -33,6 +34,26 @@ type WriteRequest struct {
|
|||||||
Reader io.Reader
|
Reader io.Reader
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// multiReaderCloser wraps an io.Reader with a Close method that closes underlying closers.
|
||||||
|
type multiReaderCloser struct {
|
||||||
|
reader io.Reader
|
||||||
|
closers []io.Closer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *multiReaderCloser) Read(p []byte) (n int, err error) {
|
||||||
|
return m.reader.Read(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *multiReaderCloser) Close() error {
|
||||||
|
var firstErr error
|
||||||
|
for _, c := range m.closers {
|
||||||
|
if err := c.Close(); err != nil && firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return firstErr
|
||||||
|
}
|
||||||
|
|
||||||
// FileProcessor handles file processing operations.
|
// FileProcessor handles file processing operations.
|
||||||
type FileProcessor struct {
|
type FileProcessor struct {
|
||||||
rootPath string
|
rootPath string
|
||||||
@@ -58,6 +79,34 @@ func NewFileProcessorWithMonitor(rootPath string, monitor *ResourceMonitor) *Fil
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkContextCancellation checks if context is cancelled and logs an error if so.
|
||||||
|
// Returns true if context is cancelled, false otherwise.
|
||||||
|
func (p *FileProcessor) checkContextCancellation(ctx context.Context, filePath, stage string) bool {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Format stage with leading space if provided
|
||||||
|
stageMsg := stage
|
||||||
|
if stage != "" {
|
||||||
|
stageMsg = " " + stage
|
||||||
|
}
|
||||||
|
gibidiutils.LogErrorf(
|
||||||
|
gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeResourceLimitTimeout,
|
||||||
|
fmt.Sprintf("file processing cancelled%s", stageMsg),
|
||||||
|
filePath,
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
"File processing cancelled%s: %s",
|
||||||
|
stageMsg,
|
||||||
|
filePath,
|
||||||
|
)
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ProcessFile reads the file at filePath and sends a formatted output to outCh.
|
// ProcessFile reads the file at filePath and sends a formatted output to outCh.
|
||||||
// It automatically chooses between loading the entire file or streaming based on file size.
|
// It automatically chooses between loading the entire file or streaming based on file size.
|
||||||
func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) {
|
func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) {
|
||||||
@@ -67,7 +116,13 @@ func ProcessFile(filePath string, outCh chan<- WriteRequest, rootPath string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ProcessFileWithMonitor processes a file using a shared resource monitor.
|
// ProcessFileWithMonitor processes a file using a shared resource monitor.
|
||||||
func ProcessFileWithMonitor(ctx context.Context, filePath string, outCh chan<- WriteRequest, rootPath string, monitor *ResourceMonitor) {
|
func ProcessFileWithMonitor(
|
||||||
|
ctx context.Context,
|
||||||
|
filePath string,
|
||||||
|
outCh chan<- WriteRequest,
|
||||||
|
rootPath string,
|
||||||
|
monitor *ResourceMonitor,
|
||||||
|
) {
|
||||||
processor := NewFileProcessorWithMonitor(rootPath, monitor)
|
processor := NewFileProcessorWithMonitor(rootPath, monitor)
|
||||||
processor.ProcessWithContext(ctx, filePath, outCh)
|
processor.ProcessWithContext(ctx, filePath, outCh)
|
||||||
}
|
}
|
||||||
@@ -86,10 +141,17 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string,
|
|||||||
|
|
||||||
// Wait for rate limiting
|
// Wait for rate limiting
|
||||||
if err := p.resourceMonitor.WaitForRateLimit(fileCtx); err != nil {
|
if err := p.resourceMonitor.WaitForRateLimit(fileCtx); err != nil {
|
||||||
if err == context.DeadlineExceeded {
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
utils.LogErrorf(
|
gibidiutils.LogErrorf(
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing timeout during rate limiting", filePath, nil),
|
gibidiutils.NewStructuredError(
|
||||||
"File processing timeout during rate limiting: %s", filePath,
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeResourceLimitTimeout,
|
||||||
|
"file processing timeout during rate limiting",
|
||||||
|
filePath,
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
"File processing timeout during rate limiting: %s",
|
||||||
|
filePath,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -103,10 +165,17 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string,
|
|||||||
|
|
||||||
// Acquire read slot for concurrent processing
|
// Acquire read slot for concurrent processing
|
||||||
if err := p.resourceMonitor.AcquireReadSlot(fileCtx); err != nil {
|
if err := p.resourceMonitor.AcquireReadSlot(fileCtx); err != nil {
|
||||||
if err == context.DeadlineExceeded {
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
utils.LogErrorf(
|
gibidiutils.LogErrorf(
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing timeout waiting for read slot", filePath, nil),
|
gibidiutils.NewStructuredError(
|
||||||
"File processing timeout waiting for read slot: %s", filePath,
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeResourceLimitTimeout,
|
||||||
|
"file processing timeout waiting for read slot",
|
||||||
|
filePath,
|
||||||
|
nil,
|
||||||
|
),
|
||||||
|
"File processing timeout waiting for read slot: %s",
|
||||||
|
filePath,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -115,7 +184,7 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string,
|
|||||||
|
|
||||||
// Check hard memory limits before processing
|
// Check hard memory limits before processing
|
||||||
if err := p.resourceMonitor.CheckHardMemoryLimit(); err != nil {
|
if err := p.resourceMonitor.CheckHardMemoryLimit(); err != nil {
|
||||||
utils.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath)
|
gibidiutils.LogErrorf(err, "Hard memory limit check failed for file: %s", filePath)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,7 +207,6 @@ func (p *FileProcessor) ProcessWithContext(ctx context.Context, filePath string,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// validateFileWithLimits checks if the file can be processed with resource limits.
|
// validateFileWithLimits checks if the file can be processed with resource limits.
|
||||||
func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath string) (os.FileInfo, error) {
|
func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath string) (os.FileInfo, error) {
|
||||||
// Check context cancellation
|
// Check context cancellation
|
||||||
@@ -150,24 +218,27 @@ func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath str
|
|||||||
|
|
||||||
fileInfo, err := os.Stat(filePath)
|
fileInfo, err := os.Stat(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
structErr := utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to stat file").WithFilePath(filePath)
|
structErr := gibidiutils.WrapError(
|
||||||
utils.LogErrorf(structErr, "Failed to stat file %s", filePath)
|
err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSAccess,
|
||||||
return nil, err
|
"failed to stat file",
|
||||||
|
).WithFilePath(filePath)
|
||||||
|
gibidiutils.LogErrorf(structErr, "Failed to stat file %s", filePath)
|
||||||
|
return nil, structErr
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check traditional size limit
|
// Check traditional size limit
|
||||||
if fileInfo.Size() > p.sizeLimit {
|
if fileInfo.Size() > p.sizeLimit {
|
||||||
context := map[string]interface{}{
|
filesizeContext := map[string]interface{}{
|
||||||
"file_size": fileInfo.Size(),
|
"file_size": fileInfo.Size(),
|
||||||
"size_limit": p.sizeLimit,
|
"size_limit": p.sizeLimit,
|
||||||
}
|
}
|
||||||
utils.LogErrorf(
|
gibidiutils.LogErrorf(
|
||||||
utils.NewStructuredError(
|
gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeValidationSize,
|
gibidiutils.CodeValidationSize,
|
||||||
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", fileInfo.Size(), p.sizeLimit),
|
fmt.Sprintf("file size (%d bytes) exceeds limit (%d bytes)", fileInfo.Size(), p.sizeLimit),
|
||||||
filePath,
|
filePath,
|
||||||
context,
|
filesizeContext,
|
||||||
),
|
),
|
||||||
"Skipping large file %s", filePath,
|
"Skipping large file %s", filePath,
|
||||||
)
|
)
|
||||||
@@ -176,7 +247,7 @@ func (p *FileProcessor) validateFileWithLimits(ctx context.Context, filePath str
|
|||||||
|
|
||||||
// Check resource limits
|
// Check resource limits
|
||||||
if err := p.resourceMonitor.ValidateFileProcessing(filePath, fileInfo.Size()); err != nil {
|
if err := p.resourceMonitor.ValidateFileProcessing(filePath, fileInfo.Size()); err != nil {
|
||||||
utils.LogErrorf(err, "Resource limit validation failed for file: %s", filePath)
|
gibidiutils.LogErrorf(err, "Resource limit validation failed for file: %s", filePath)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,66 +263,54 @@ func (p *FileProcessor) getRelativePath(filePath string) string {
|
|||||||
return relPath
|
return relPath
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// processInMemoryWithContext loads the entire file into memory with context awareness.
|
// processInMemoryWithContext loads the entire file into memory with context awareness.
|
||||||
func (p *FileProcessor) processInMemoryWithContext(ctx context.Context, filePath, relPath string, outCh chan<- WriteRequest) {
|
func (p *FileProcessor) processInMemoryWithContext(
|
||||||
|
ctx context.Context,
|
||||||
|
filePath, relPath string,
|
||||||
|
outCh chan<- WriteRequest,
|
||||||
|
) {
|
||||||
// Check context before reading
|
// Check context before reading
|
||||||
select {
|
if p.checkContextCancellation(ctx, filePath, "") {
|
||||||
case <-ctx.Done():
|
|
||||||
utils.LogErrorf(
|
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled", filePath, nil),
|
|
||||||
"File processing cancelled: %s", filePath,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
content, err := os.ReadFile(filePath) // #nosec G304 - filePath is validated by walker
|
// #nosec G304 - filePath is validated by walker
|
||||||
|
content, err := os.ReadFile(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to read file").WithFilePath(filePath)
|
structErr := gibidiutils.WrapError(
|
||||||
utils.LogErrorf(structErr, "Failed to read file %s", filePath)
|
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingFileRead,
|
||||||
|
"failed to read file",
|
||||||
|
).WithFilePath(filePath)
|
||||||
|
gibidiutils.LogErrorf(structErr, "Failed to read file %s", filePath)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check context again after reading
|
// Check context again after reading
|
||||||
select {
|
if p.checkContextCancellation(ctx, filePath, "after read") {
|
||||||
case <-ctx.Done():
|
|
||||||
utils.LogErrorf(
|
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled after read", filePath, nil),
|
|
||||||
"File processing cancelled after read: %s", filePath,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to send the result, but respect context cancellation
|
// Check context before sending output
|
||||||
select {
|
if p.checkContextCancellation(ctx, filePath, "before output") {
|
||||||
case <-ctx.Done():
|
|
||||||
utils.LogErrorf(
|
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "file processing cancelled before output", filePath, nil),
|
|
||||||
"File processing cancelled before output: %s", filePath,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
case outCh <- WriteRequest{
|
}
|
||||||
|
|
||||||
|
outCh <- WriteRequest{
|
||||||
Path: relPath,
|
Path: relPath,
|
||||||
Content: p.formatContent(relPath, string(content)),
|
Content: p.formatContent(relPath, string(content)),
|
||||||
IsStream: false,
|
IsStream: false,
|
||||||
}:
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// processStreamingWithContext creates a streaming reader for large files with context awareness.
|
// processStreamingWithContext creates a streaming reader for large files with context awareness.
|
||||||
func (p *FileProcessor) processStreamingWithContext(ctx context.Context, filePath, relPath string, outCh chan<- WriteRequest) {
|
func (p *FileProcessor) processStreamingWithContext(
|
||||||
|
ctx context.Context,
|
||||||
|
filePath, relPath string,
|
||||||
|
outCh chan<- WriteRequest,
|
||||||
|
) {
|
||||||
// Check context before creating reader
|
// Check context before creating reader
|
||||||
select {
|
if p.checkContextCancellation(ctx, filePath, "before streaming") {
|
||||||
case <-ctx.Done():
|
|
||||||
utils.LogErrorf(
|
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "streaming processing cancelled", filePath, nil),
|
|
||||||
"Streaming processing cancelled: %s", filePath,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
reader := p.createStreamReaderWithContext(ctx, filePath, relPath)
|
reader := p.createStreamReaderWithContext(ctx, filePath, relPath)
|
||||||
@@ -259,43 +318,47 @@ func (p *FileProcessor) processStreamingWithContext(ctx context.Context, filePat
|
|||||||
return // Error already logged
|
return // Error already logged
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to send the result, but respect context cancellation
|
// Check context before sending output
|
||||||
select {
|
if p.checkContextCancellation(ctx, filePath, "before streaming output") {
|
||||||
case <-ctx.Done():
|
// Close the reader to prevent file descriptor leak
|
||||||
utils.LogErrorf(
|
if closer, ok := reader.(io.Closer); ok {
|
||||||
utils.NewStructuredError(utils.ErrorTypeValidation, utils.CodeResourceLimitTimeout, "streaming processing cancelled before output", filePath, nil),
|
_ = closer.Close()
|
||||||
"Streaming processing cancelled before output: %s", filePath,
|
}
|
||||||
)
|
|
||||||
return
|
return
|
||||||
case outCh <- WriteRequest{
|
}
|
||||||
|
|
||||||
|
outCh <- WriteRequest{
|
||||||
Path: relPath,
|
Path: relPath,
|
||||||
Content: "", // Empty since content is in Reader
|
Content: "", // Empty since content is in Reader
|
||||||
IsStream: true,
|
IsStream: true,
|
||||||
Reader: reader,
|
Reader: reader,
|
||||||
}:
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// createStreamReaderWithContext creates a reader that combines header and file content with context awareness.
|
// createStreamReaderWithContext creates a reader that combines header and file content with context awareness.
|
||||||
func (p *FileProcessor) createStreamReaderWithContext(ctx context.Context, filePath, relPath string) io.Reader {
|
func (p *FileProcessor) createStreamReaderWithContext(ctx context.Context, filePath, relPath string) io.Reader {
|
||||||
// Check context before opening file
|
// Check context before opening file
|
||||||
select {
|
if p.checkContextCancellation(ctx, filePath, "before opening file") {
|
||||||
case <-ctx.Done():
|
|
||||||
return nil
|
return nil
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Open(filePath) // #nosec G304 - filePath is validated by walker
|
// #nosec G304 - filePath is validated by walker
|
||||||
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
structErr := utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingFileRead, "failed to open file for streaming").WithFilePath(filePath)
|
structErr := gibidiutils.WrapError(
|
||||||
utils.LogErrorf(structErr, "Failed to open file for streaming %s", filePath)
|
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingFileRead,
|
||||||
|
"failed to open file for streaming",
|
||||||
|
).WithFilePath(filePath)
|
||||||
|
gibidiutils.LogErrorf(structErr, "Failed to open file for streaming %s", filePath)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Note: file will be closed by the writer
|
|
||||||
|
|
||||||
header := p.formatHeader(relPath)
|
header := p.formatHeader(relPath)
|
||||||
return io.MultiReader(header, file)
|
// Wrap in multiReaderCloser to ensure file is closed even on cancellation
|
||||||
|
return &multiReaderCloser{
|
||||||
|
reader: io.MultiReader(header, file),
|
||||||
|
closers: []io.Closer{file},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// formatContent formats the file content with header.
|
// formatContent formats the file content with header.
|
||||||
|
|||||||
@@ -51,9 +51,11 @@ func (rm *ResourceMonitor) CreateFileProcessingContext(parent context.Context) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CreateOverallProcessingContext creates a context with overall processing timeout.
|
// CreateOverallProcessingContext creates a context with overall processing timeout.
|
||||||
func (rm *ResourceMonitor) CreateOverallProcessingContext(parent context.Context) (context.Context, context.CancelFunc) {
|
func (rm *ResourceMonitor) CreateOverallProcessingContext(
|
||||||
|
parent context.Context,
|
||||||
|
) (context.Context, context.CancelFunc) {
|
||||||
if !rm.enabled || rm.overallTimeout <= 0 {
|
if !rm.enabled || rm.overallTimeout <= 0 {
|
||||||
return parent, func() {}
|
return parent, func() {}
|
||||||
}
|
}
|
||||||
return context.WithTimeout(parent, rm.overallTimeout)
|
return context.WithTimeout(parent, rm.overallTimeout)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) {
|
|||||||
t.Errorf("Expected no error for second read slot, got %v", err)
|
t.Errorf("Expected no error for second read slot, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Third read slot should timeout (context deadline exceeded)
|
// Third read slot should time out (context deadline exceeded)
|
||||||
err = rm.AcquireReadSlot(ctx)
|
err = rm.AcquireReadSlot(ctx)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("Expected timeout error for third read slot, got nil")
|
t.Error("Expected timeout error for third read slot, got nil")
|
||||||
@@ -43,11 +43,11 @@ func TestResourceMonitor_ConcurrentReadsLimit(t *testing.T) {
|
|||||||
|
|
||||||
// Release one slot and try again
|
// Release one slot and try again
|
||||||
rm.ReleaseReadSlot()
|
rm.ReleaseReadSlot()
|
||||||
|
|
||||||
// Create new context for the next attempt
|
// Create new context for the next attempt
|
||||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
ctx2, cancel2 := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||||
defer cancel2()
|
defer cancel2()
|
||||||
|
|
||||||
err = rm.AcquireReadSlot(ctx2)
|
err = rm.AcquireReadSlot(ctx2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Expected no error after releasing a slot, got %v", err)
|
t.Errorf("Expected no error after releasing a slot, got %v", err)
|
||||||
@@ -92,4 +92,4 @@ func TestResourceMonitor_TimeoutContexts(t *testing.T) {
|
|||||||
} else if time.Until(deadline) > 2*time.Second+100*time.Millisecond {
|
} else if time.Until(deadline) > 2*time.Second+100*time.Millisecond {
|
||||||
t.Error("Overall processing timeout appears to be too long")
|
t.Error("Overall processing timeout appears to be too long")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,4 +78,4 @@ func TestResourceMonitor_Integration(t *testing.T) {
|
|||||||
|
|
||||||
// Test resource limit logging
|
// Test resource limit logging
|
||||||
rm.LogResourceInfo()
|
rm.LogResourceInfo()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RecordFileProcessed records that a file has been successfully processed.
|
// RecordFileProcessed records that a file has been successfully processed.
|
||||||
@@ -55,7 +57,7 @@ func (rm *ResourceMonitor) GetMetrics() ResourceMetrics {
|
|||||||
ProcessingDuration: duration,
|
ProcessingDuration: duration,
|
||||||
AverageFileSize: avgFileSize,
|
AverageFileSize: avgFileSize,
|
||||||
ProcessingRate: processingRate,
|
ProcessingRate: processingRate,
|
||||||
MemoryUsageMB: int64(m.Alloc) / 1024 / 1024,
|
MemoryUsageMB: gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0) / 1024 / 1024,
|
||||||
MaxMemoryUsageMB: int64(rm.hardMemoryLimitMB),
|
MaxMemoryUsageMB: int64(rm.hardMemoryLimitMB),
|
||||||
ViolationsDetected: violations,
|
ViolationsDetected: violations,
|
||||||
DegradationActive: rm.degradationActive,
|
DegradationActive: rm.degradationActive,
|
||||||
@@ -67,8 +69,13 @@ func (rm *ResourceMonitor) GetMetrics() ResourceMetrics {
|
|||||||
// LogResourceInfo logs current resource limit configuration.
|
// LogResourceInfo logs current resource limit configuration.
|
||||||
func (rm *ResourceMonitor) LogResourceInfo() {
|
func (rm *ResourceMonitor) LogResourceInfo() {
|
||||||
if rm.enabled {
|
if rm.enabled {
|
||||||
logrus.Infof("Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds",
|
logrus.Infof(
|
||||||
rm.maxFiles, rm.maxTotalSize/1024/1024, int(rm.fileProcessingTimeout.Seconds()), int(rm.overallTimeout.Seconds()))
|
"Resource limits enabled: maxFiles=%d, maxTotalSize=%dMB, fileTimeout=%ds, overallTimeout=%ds",
|
||||||
|
rm.maxFiles,
|
||||||
|
rm.maxTotalSize/1024/1024,
|
||||||
|
int(rm.fileProcessingTimeout.Seconds()),
|
||||||
|
int(rm.overallTimeout.Seconds()),
|
||||||
|
)
|
||||||
logrus.Infof("Resource limits: maxConcurrentReads=%d, rateLimitFPS=%d, hardMemoryMB=%d",
|
logrus.Infof("Resource limits: maxConcurrentReads=%d, rateLimitFPS=%d, hardMemoryMB=%d",
|
||||||
rm.maxConcurrentReads, rm.rateLimitFilesPerSec, rm.hardMemoryLimitMB)
|
rm.maxConcurrentReads, rm.rateLimitFilesPerSec, rm.hardMemoryLimitMB)
|
||||||
logrus.Infof("Resource features: gracefulDegradation=%v, monitoring=%v",
|
logrus.Infof("Resource features: gracefulDegradation=%v, monitoring=%v",
|
||||||
@@ -76,4 +83,4 @@ func (rm *ResourceMonitor) LogResourceInfo() {
|
|||||||
} else {
|
} else {
|
||||||
logrus.Info("Resource limits disabled")
|
logrus.Info("Resource limits disabled")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,4 +46,4 @@ func TestResourceMonitor_Metrics(t *testing.T) {
|
|||||||
if !metrics.LastUpdated.After(time.Now().Add(-time.Second)) {
|
if !metrics.LastUpdated.After(time.Now().Add(-time.Second)) {
|
||||||
t.Error("Expected recent LastUpdated timestamp")
|
t.Error("Expected recent LastUpdated timestamp")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,4 +33,4 @@ func (rm *ResourceMonitor) rateLimiterRefill() {
|
|||||||
// Channel is full, skip
|
// Channel is full, skip
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,4 +37,4 @@ func TestResourceMonitor_RateLimiting(t *testing.T) {
|
|||||||
if duration < 200*time.Millisecond {
|
if duration < 200*time.Millisecond {
|
||||||
t.Logf("Rate limiting may not be working as expected, took only %v", duration)
|
t.Logf("Rate limiting may not be working as expected, took only %v", duration)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ func (rm *ResourceMonitor) Close() {
|
|||||||
if rm.rateLimiter != nil {
|
if rm.rateLimiter != nil {
|
||||||
rm.rateLimiter.Stop()
|
rm.rateLimiter.Stop()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,9 +100,9 @@ func NewResourceMonitor() *ResourceMonitor {
|
|||||||
}
|
}
|
||||||
rateLimitFull:
|
rateLimitFull:
|
||||||
|
|
||||||
// Start rate limiter refill goroutine
|
// Start rate limiter refill goroutine
|
||||||
go rm.rateLimiterRefill()
|
go rm.rateLimiterRefill()
|
||||||
}
|
}
|
||||||
|
|
||||||
return rm
|
return rm
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ func TestResourceMonitor_NewResourceMonitor(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if rm.fileProcessingTimeout != time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second {
|
if rm.fileProcessingTimeout != time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second {
|
||||||
t.Errorf("Expected fileProcessingTimeout to be %v, got %v",
|
t.Errorf("Expected fileProcessingTimeout to be %v, got %v",
|
||||||
time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second, rm.fileProcessingTimeout)
|
time.Duration(config.DefaultFileProcessingTimeoutSec)*time.Second, rm.fileProcessingTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,4 +71,4 @@ func TestResourceMonitor_DisabledResourceLimits(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Expected no error when rate limiting disabled, got %v", err)
|
t.Errorf("Expected no error when rate limiting disabled, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ValidateFileProcessing checks if a file can be processed based on resource limits.
|
// ValidateFileProcessing checks if a file can be processed based on resource limits.
|
||||||
@@ -21,9 +21,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
|
|||||||
|
|
||||||
// Check if emergency stop is active
|
// Check if emergency stop is active
|
||||||
if rm.emergencyStopRequested {
|
if rm.emergencyStopRequested {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitMemory,
|
gibidiutils.CodeResourceLimitMemory,
|
||||||
"processing stopped due to emergency memory condition",
|
"processing stopped due to emergency memory condition",
|
||||||
filePath,
|
filePath,
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -35,9 +35,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
|
|||||||
// Check file count limit
|
// Check file count limit
|
||||||
currentFiles := atomic.LoadInt64(&rm.filesProcessed)
|
currentFiles := atomic.LoadInt64(&rm.filesProcessed)
|
||||||
if int(currentFiles) >= rm.maxFiles {
|
if int(currentFiles) >= rm.maxFiles {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitFiles,
|
gibidiutils.CodeResourceLimitFiles,
|
||||||
"maximum file count limit exceeded",
|
"maximum file count limit exceeded",
|
||||||
filePath,
|
filePath,
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -50,9 +50,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
|
|||||||
// Check total size limit
|
// Check total size limit
|
||||||
currentTotalSize := atomic.LoadInt64(&rm.totalSizeProcessed)
|
currentTotalSize := atomic.LoadInt64(&rm.totalSizeProcessed)
|
||||||
if currentTotalSize+fileSize > rm.maxTotalSize {
|
if currentTotalSize+fileSize > rm.maxTotalSize {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitTotalSize,
|
gibidiutils.CodeResourceLimitTotalSize,
|
||||||
"maximum total size limit would be exceeded",
|
"maximum total size limit would be exceeded",
|
||||||
filePath,
|
filePath,
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -65,9 +65,9 @@ func (rm *ResourceMonitor) ValidateFileProcessing(filePath string, fileSize int6
|
|||||||
|
|
||||||
// Check overall timeout
|
// Check overall timeout
|
||||||
if time.Since(rm.startTime) > rm.overallTimeout {
|
if time.Since(rm.startTime) > rm.overallTimeout {
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitTimeout,
|
gibidiutils.CodeResourceLimitTimeout,
|
||||||
"overall processing timeout exceeded",
|
"overall processing timeout exceeded",
|
||||||
filePath,
|
filePath,
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -88,7 +88,7 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error {
|
|||||||
|
|
||||||
var m runtime.MemStats
|
var m runtime.MemStats
|
||||||
runtime.ReadMemStats(&m)
|
runtime.ReadMemStats(&m)
|
||||||
currentMemory := int64(m.Alloc)
|
currentMemory := gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0)
|
||||||
|
|
||||||
if currentMemory > rm.hardMemoryLimitBytes {
|
if currentMemory > rm.hardMemoryLimitBytes {
|
||||||
rm.mu.Lock()
|
rm.mu.Lock()
|
||||||
@@ -108,14 +108,14 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error {
|
|||||||
|
|
||||||
// Check again after GC
|
// Check again after GC
|
||||||
runtime.ReadMemStats(&m)
|
runtime.ReadMemStats(&m)
|
||||||
currentMemory = int64(m.Alloc)
|
currentMemory = gibidiutils.SafeUint64ToInt64WithDefault(m.Alloc, 0)
|
||||||
|
|
||||||
if currentMemory > rm.hardMemoryLimitBytes {
|
if currentMemory > rm.hardMemoryLimitBytes {
|
||||||
// Still over limit, activate emergency stop
|
// Still over limit, activate emergency stop
|
||||||
rm.emergencyStopRequested = true
|
rm.emergencyStopRequested = true
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitMemory,
|
gibidiutils.CodeResourceLimitMemory,
|
||||||
"hard memory limit exceeded, emergency stop activated",
|
"hard memory limit exceeded, emergency stop activated",
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -124,16 +124,15 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error {
|
|||||||
"emergency_stop": true,
|
"emergency_stop": true,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
} else {
|
|
||||||
// Memory freed by GC, continue with degradation
|
|
||||||
rm.degradationActive = true
|
|
||||||
logrus.Info("Memory freed by garbage collection, continuing with degradation mode")
|
|
||||||
}
|
}
|
||||||
|
// Memory freed by GC, continue with degradation
|
||||||
|
rm.degradationActive = true
|
||||||
|
logrus.Info("Memory freed by garbage collection, continuing with degradation mode")
|
||||||
} else {
|
} else {
|
||||||
// No graceful degradation, hard stop
|
// No graceful degradation, hard stop
|
||||||
return utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeResourceLimitMemory,
|
gibidiutils.CodeResourceLimitMemory,
|
||||||
"hard memory limit exceeded",
|
"hard memory limit exceeded",
|
||||||
"",
|
"",
|
||||||
map[string]interface{}{
|
map[string]interface{}{
|
||||||
@@ -145,4 +144,4 @@ func (rm *ResourceMonitor) CheckHardMemoryLimit() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
package fileproc
|
package fileproc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
"github.com/ivuorinen/gibidify/testutil"
|
"github.com/ivuorinen/gibidify/testutil"
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestResourceMonitor_FileCountLimit(t *testing.T) {
|
func TestResourceMonitor_FileCountLimit(t *testing.T) {
|
||||||
@@ -40,11 +41,12 @@ func TestResourceMonitor_FileCountLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify it's the correct error type
|
// Verify it's the correct error type
|
||||||
structErr, ok := err.(*utils.StructuredError)
|
var structErr *gibidiutils.StructuredError
|
||||||
|
ok := errors.As(err, &structErr)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Errorf("Expected StructuredError, got %T", err)
|
t.Errorf("Expected StructuredError, got %T", err)
|
||||||
} else if structErr.Code != utils.CodeResourceLimitFiles {
|
} else if structErr.Code != gibidiutils.CodeResourceLimitFiles {
|
||||||
t.Errorf("Expected error code %s, got %s", utils.CodeResourceLimitFiles, structErr.Code)
|
t.Errorf("Expected error code %s, got %s", gibidiutils.CodeResourceLimitFiles, structErr.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,10 +81,11 @@ func TestResourceMonitor_TotalSizeLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify it's the correct error type
|
// Verify it's the correct error type
|
||||||
structErr, ok := err.(*utils.StructuredError)
|
var structErr *gibidiutils.StructuredError
|
||||||
|
ok := errors.As(err, &structErr)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Errorf("Expected StructuredError, got %T", err)
|
t.Errorf("Expected StructuredError, got %T", err)
|
||||||
} else if structErr.Code != utils.CodeResourceLimitTotalSize {
|
} else if structErr.Code != gibidiutils.CodeResourceLimitTotalSize {
|
||||||
t.Errorf("Expected error code %s, got %s", utils.CodeResourceLimitTotalSize, structErr.Code)
|
t.Errorf("Expected error code %s, got %s", gibidiutils.CodeResourceLimitTotalSize, structErr.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
12
fileproc/test_constants.go
Normal file
12
fileproc/test_constants.go
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
package fileproc
|
||||||
|
|
||||||
|
// Test constants to avoid duplication in test files.
|
||||||
|
// These constants are used across multiple test files in the fileproc package.
|
||||||
|
const (
|
||||||
|
// Backpressure configuration keys
|
||||||
|
testBackpressureEnabled = "backpressure.enabled"
|
||||||
|
testBackpressureMaxMemory = "backpressure.maxMemoryUsage"
|
||||||
|
testBackpressureMemoryCheck = "backpressure.memoryCheckInterval"
|
||||||
|
testBackpressureMaxFiles = "backpressure.maxPendingFiles"
|
||||||
|
testBackpressureMaxWrites = "backpressure.maxPendingWrites"
|
||||||
|
)
|
||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Walker defines an interface for scanning directories.
|
// Walker defines an interface for scanning directories.
|
||||||
@@ -30,9 +30,12 @@ func NewProdWalker() *ProdWalker {
|
|||||||
// Walk scans the given root directory recursively and returns a slice of file paths
|
// Walk scans the given root directory recursively and returns a slice of file paths
|
||||||
// that are not ignored based on .gitignore/.ignore files, the configuration, or the default binary/image filter.
|
// that are not ignored based on .gitignore/.ignore files, the configuration, or the default binary/image filter.
|
||||||
func (w *ProdWalker) Walk(root string) ([]string, error) {
|
func (w *ProdWalker) Walk(root string) ([]string, error) {
|
||||||
absRoot, err := utils.GetAbsolutePath(root)
|
absRoot, err := gibidiutils.GetAbsolutePath(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSPathResolution, "failed to resolve root path").WithFilePath(root)
|
return nil, gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSPathResolution,
|
||||||
|
"failed to resolve root path",
|
||||||
|
).WithFilePath(root)
|
||||||
}
|
}
|
||||||
return w.walkDir(absRoot, []ignoreRule{})
|
return w.walkDir(absRoot, []ignoreRule{})
|
||||||
}
|
}
|
||||||
@@ -47,7 +50,10 @@ func (w *ProdWalker) walkDir(currentDir string, parentRules []ignoreRule) ([]str
|
|||||||
|
|
||||||
entries, err := os.ReadDir(currentDir)
|
entries, err := os.ReadDir(currentDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeFileSystem, utils.CodeFSAccess, "failed to read directory").WithFilePath(currentDir)
|
return nil, gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeFileSystem, gibidiutils.CodeFSAccess,
|
||||||
|
"failed to read directory",
|
||||||
|
).WithFilePath(currentDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
rules := loadIgnoreRules(currentDir, parentRules)
|
rules := loadIgnoreRules(currentDir, parentRules)
|
||||||
@@ -63,7 +69,10 @@ func (w *ProdWalker) walkDir(currentDir string, parentRules []ignoreRule) ([]str
|
|||||||
if entry.IsDir() {
|
if entry.IsDir() {
|
||||||
subFiles, err := w.walkDir(fullPath, rules)
|
subFiles, err := w.walkDir(fullPath, rules)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, utils.WrapError(err, utils.ErrorTypeProcessing, utils.CodeProcessingTraversal, "failed to traverse subdirectory").WithFilePath(fullPath)
|
return nil, gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeProcessing, gibidiutils.CodeProcessingTraversal,
|
||||||
|
"failed to traverse subdirectory",
|
||||||
|
).WithFilePath(fullPath)
|
||||||
}
|
}
|
||||||
results = append(results, subFiles...)
|
results = append(results, subFiles...)
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -61,6 +61,8 @@ func TestProdWalkerBinaryCheck(t *testing.T) {
|
|||||||
|
|
||||||
// Reset FileTypeRegistry to ensure clean state
|
// Reset FileTypeRegistry to ensure clean state
|
||||||
fileproc.ResetRegistryForTesting()
|
fileproc.ResetRegistryForTesting()
|
||||||
|
// Ensure cleanup runs even if test fails
|
||||||
|
t.Cleanup(fileproc.ResetRegistryForTesting)
|
||||||
|
|
||||||
// Run walker
|
// Run walker
|
||||||
w := fileproc.NewProdWalker()
|
w := fileproc.NewProdWalker()
|
||||||
|
|||||||
@@ -5,30 +5,100 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// StartWriter writes the output in the specified format with memory optimization.
|
// WriterConfig holds configuration for the writer.
|
||||||
func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, format, prefix, suffix string) {
|
type WriterConfig struct {
|
||||||
switch format {
|
Format string
|
||||||
case "markdown":
|
Prefix string
|
||||||
startMarkdownWriter(outFile, writeCh, done, prefix, suffix)
|
Suffix string
|
||||||
case "json":
|
}
|
||||||
startJSONWriter(outFile, writeCh, done, prefix, suffix)
|
|
||||||
case "yaml":
|
// Validate checks if the WriterConfig is valid.
|
||||||
startYAMLWriter(outFile, writeCh, done, prefix, suffix)
|
func (c WriterConfig) Validate() error {
|
||||||
|
if c.Format == "" {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
"format cannot be empty",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch c.Format {
|
||||||
|
case "markdown", "json", "yaml":
|
||||||
|
return nil
|
||||||
default:
|
default:
|
||||||
context := map[string]interface{}{
|
context := map[string]any{
|
||||||
"format": format,
|
"format": c.Format,
|
||||||
}
|
}
|
||||||
err := utils.NewStructuredError(
|
return gibidiutils.NewStructuredError(
|
||||||
utils.ErrorTypeValidation,
|
gibidiutils.ErrorTypeValidation,
|
||||||
utils.CodeValidationFormat,
|
gibidiutils.CodeValidationFormat,
|
||||||
fmt.Sprintf("unsupported format: %s", format),
|
fmt.Sprintf("unsupported format: %s", c.Format),
|
||||||
"",
|
"",
|
||||||
context,
|
context,
|
||||||
)
|
)
|
||||||
utils.LogError("Failed to encode output", err)
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StartWriter writes the output in the specified format with memory optimization.
|
||||||
|
func StartWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, config WriterConfig) {
|
||||||
|
// Validate config
|
||||||
|
if err := config.Validate(); err != nil {
|
||||||
|
gibidiutils.LogError("Invalid writer configuration", err)
|
||||||
|
close(done)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate outFile is not nil
|
||||||
|
if outFile == nil {
|
||||||
|
err := gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileWrite,
|
||||||
|
"output file is nil",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
gibidiutils.LogError("Failed to write output", err)
|
||||||
|
close(done)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate outFile is accessible
|
||||||
|
if _, err := outFile.Stat(); err != nil {
|
||||||
|
structErr := gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOFileWrite,
|
||||||
|
"failed to stat output file",
|
||||||
|
)
|
||||||
|
gibidiutils.LogError("Failed to validate output file", structErr)
|
||||||
|
close(done)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
switch config.Format {
|
||||||
|
case "markdown":
|
||||||
|
startMarkdownWriter(outFile, writeCh, done, config.Prefix, config.Suffix)
|
||||||
|
case "json":
|
||||||
|
startJSONWriter(outFile, writeCh, done, config.Prefix, config.Suffix)
|
||||||
|
case "yaml":
|
||||||
|
startYAMLWriter(outFile, writeCh, done, config.Prefix, config.Suffix)
|
||||||
|
default:
|
||||||
|
context := map[string]interface{}{
|
||||||
|
"format": config.Format,
|
||||||
|
}
|
||||||
|
err := gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationFormat,
|
||||||
|
fmt.Sprintf("unsupported format: %s", config.Format),
|
||||||
|
"",
|
||||||
|
context,
|
||||||
|
)
|
||||||
|
gibidiutils.LogError("Failed to encode output", err)
|
||||||
close(done)
|
close(done)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,7 +68,11 @@ func runWriterTest(t *testing.T, format string) []byte {
|
|||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
fileproc.StartWriter(outFile, writeCh, doneCh, format, "PREFIX", "SUFFIX")
|
fileproc.StartWriter(outFile, writeCh, doneCh, fileproc.WriterConfig{
|
||||||
|
Format: format,
|
||||||
|
Prefix: "PREFIX",
|
||||||
|
Suffix: "SUFFIX",
|
||||||
|
})
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Wait until writer signals completion
|
// Wait until writer signals completion
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/ivuorinen/gibidify/utils"
|
"github.com/ivuorinen/gibidify/gibidiutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// YAMLWriter handles YAML format output with streaming support.
|
// YAMLWriter handles YAML format output with streaming support.
|
||||||
@@ -20,11 +21,151 @@ func NewYAMLWriter(outFile *os.File) *YAMLWriter {
|
|||||||
return &YAMLWriter{outFile: outFile}
|
return &YAMLWriter{outFile: outFile}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxPathLength = 4096 // Maximum total path length
|
||||||
|
maxFilenameLength = 255 // Maximum individual filename component length
|
||||||
|
)
|
||||||
|
|
||||||
|
// validatePathComponents validates individual path components for security issues.
|
||||||
|
func validatePathComponents(trimmed, cleaned string, components []string) error {
|
||||||
|
for i, component := range components {
|
||||||
|
// Reject path components that are exactly ".." (path traversal)
|
||||||
|
if component == ".." {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path traversal not allowed",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{
|
||||||
|
"path": trimmed,
|
||||||
|
"cleaned": cleaned,
|
||||||
|
"invalid_component": component,
|
||||||
|
"component_index": i,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reject empty components (e.g., from "foo//bar")
|
||||||
|
if component == "" && i > 0 && i < len(components)-1 {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path contains empty component",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{
|
||||||
|
"path": trimmed,
|
||||||
|
"cleaned": cleaned,
|
||||||
|
"component_index": i,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enforce maximum filename length for each component
|
||||||
|
if len(component) > maxFilenameLength {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path component exceeds maximum length",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{
|
||||||
|
"component": component,
|
||||||
|
"component_length": len(component),
|
||||||
|
"max_length": maxFilenameLength,
|
||||||
|
"component_index": i,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validatePath validates and sanitizes a file path for safe output.
|
||||||
|
// It rejects absolute paths, path traversal attempts, empty paths, and overly long paths.
|
||||||
|
func validatePath(path string) error {
|
||||||
|
// Reject empty paths
|
||||||
|
trimmed := strings.TrimSpace(path)
|
||||||
|
if trimmed == "" {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationRequired,
|
||||||
|
"file path cannot be empty",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enforce maximum path length to prevent resource abuse
|
||||||
|
if len(trimmed) > maxPathLength {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path exceeds maximum length",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{
|
||||||
|
"path_length": len(trimmed),
|
||||||
|
"max_length": maxPathLength,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reject absolute paths
|
||||||
|
if filepath.IsAbs(trimmed) {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"absolute paths are not allowed",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{"path": trimmed},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate original trimmed path components before cleaning
|
||||||
|
origComponents := strings.Split(filepath.ToSlash(trimmed), "/")
|
||||||
|
for _, comp := range origComponents {
|
||||||
|
if comp == "" || comp == "." || comp == ".." {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"invalid or traversal path component in original path",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{"path": trimmed, "component": comp},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean the path to normalize it
|
||||||
|
cleaned := filepath.Clean(trimmed)
|
||||||
|
|
||||||
|
// After cleaning, ensure it's still relative and doesn't start with /
|
||||||
|
if filepath.IsAbs(cleaned) || strings.HasPrefix(cleaned, "/") {
|
||||||
|
return gibidiutils.NewStructuredError(
|
||||||
|
gibidiutils.ErrorTypeValidation,
|
||||||
|
gibidiutils.CodeValidationPath,
|
||||||
|
"path must be relative",
|
||||||
|
trimmed,
|
||||||
|
map[string]any{"path": trimmed, "cleaned": cleaned},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Split into components and validate each one
|
||||||
|
// Use ToSlash to normalize for cross-platform validation
|
||||||
|
components := strings.Split(filepath.ToSlash(cleaned), "/")
|
||||||
|
return validatePathComponents(trimmed, cleaned, components)
|
||||||
|
}
|
||||||
|
|
||||||
// Start writes the YAML header.
|
// Start writes the YAML header.
|
||||||
func (w *YAMLWriter) Start(prefix, suffix string) error {
|
func (w *YAMLWriter) Start(prefix, suffix string) error {
|
||||||
// Write YAML header
|
// Write YAML header
|
||||||
if _, err := fmt.Fprintf(w.outFile, "prefix: %s\nsuffix: %s\nfiles:\n", yamlQuoteString(prefix), yamlQuoteString(suffix)); err != nil {
|
if _, err := fmt.Fprintf(
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML header")
|
w.outFile, "prefix: %s\nsuffix: %s\nfiles:\n",
|
||||||
|
gibidiutils.EscapeForYAML(prefix), gibidiutils.EscapeForYAML(suffix),
|
||||||
|
); err != nil {
|
||||||
|
return gibidiutils.WrapError(
|
||||||
|
err,
|
||||||
|
gibidiutils.ErrorTypeIO,
|
||||||
|
gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write YAML header",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -44,13 +185,32 @@ func (w *YAMLWriter) Close() error {
|
|||||||
|
|
||||||
// writeStreaming writes a large file as YAML in streaming chunks.
|
// writeStreaming writes a large file as YAML in streaming chunks.
|
||||||
func (w *YAMLWriter) writeStreaming(req WriteRequest) error {
|
func (w *YAMLWriter) writeStreaming(req WriteRequest) error {
|
||||||
defer w.closeReader(req.Reader, req.Path)
|
// Validate path before using it
|
||||||
|
if err := validatePath(req.Path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for nil reader
|
||||||
|
if req.Reader == nil {
|
||||||
|
return gibidiutils.WrapError(
|
||||||
|
nil, gibidiutils.ErrorTypeValidation, gibidiutils.CodeValidationRequired,
|
||||||
|
"nil reader in write request",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer gibidiutils.SafeCloseReader(req.Reader, req.Path)
|
||||||
|
|
||||||
language := detectLanguage(req.Path)
|
language := detectLanguage(req.Path)
|
||||||
|
|
||||||
// Write YAML file entry start
|
// Write YAML file entry start
|
||||||
if _, err := fmt.Fprintf(w.outFile, " - path: %s\n language: %s\n content: |\n", yamlQuoteString(req.Path), language); err != nil {
|
if _, err := fmt.Fprintf(
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML file start").WithFilePath(req.Path)
|
w.outFile, " - path: %s\n language: %s\n content: |\n",
|
||||||
|
gibidiutils.EscapeForYAML(req.Path), language,
|
||||||
|
); err != nil {
|
||||||
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write YAML file start",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream content with YAML indentation
|
// Stream content with YAML indentation
|
||||||
@@ -59,6 +219,11 @@ func (w *YAMLWriter) writeStreaming(req WriteRequest) error {
|
|||||||
|
|
||||||
// writeInline writes a small file directly as YAML.
|
// writeInline writes a small file directly as YAML.
|
||||||
func (w *YAMLWriter) writeInline(req WriteRequest) error {
|
func (w *YAMLWriter) writeInline(req WriteRequest) error {
|
||||||
|
// Validate path before using it
|
||||||
|
if err := validatePath(req.Path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
language := detectLanguage(req.Path)
|
language := detectLanguage(req.Path)
|
||||||
fileData := FileData{
|
fileData := FileData{
|
||||||
Path: req.Path,
|
Path: req.Path,
|
||||||
@@ -67,15 +232,24 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Write YAML entry
|
// Write YAML entry
|
||||||
if _, err := fmt.Fprintf(w.outFile, " - path: %s\n language: %s\n content: |\n", yamlQuoteString(fileData.Path), fileData.Language); err != nil {
|
if _, err := fmt.Fprintf(
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML entry start").WithFilePath(req.Path)
|
w.outFile, " - path: %s\n language: %s\n content: |\n",
|
||||||
|
gibidiutils.EscapeForYAML(fileData.Path), fileData.Language,
|
||||||
|
); err != nil {
|
||||||
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write YAML entry start",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write indented content
|
// Write indented content
|
||||||
lines := strings.Split(fileData.Content, "\n")
|
lines := strings.Split(fileData.Content, "\n")
|
||||||
for _, line := range lines {
|
for _, line := range lines {
|
||||||
if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil {
|
if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML content line").WithFilePath(req.Path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write YAML content line",
|
||||||
|
).WithFilePath(req.Path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,43 +259,29 @@ func (w *YAMLWriter) writeInline(req WriteRequest) error {
|
|||||||
// streamYAMLContent streams content with YAML indentation.
|
// streamYAMLContent streams content with YAML indentation.
|
||||||
func (w *YAMLWriter) streamYAMLContent(reader io.Reader, path string) error {
|
func (w *YAMLWriter) streamYAMLContent(reader io.Reader, path string) error {
|
||||||
scanner := bufio.NewScanner(reader)
|
scanner := bufio.NewScanner(reader)
|
||||||
|
// Increase buffer size to handle long lines (up to 10MB per line)
|
||||||
|
buf := make([]byte, 0, 64*1024)
|
||||||
|
scanner.Buffer(buf, 10*1024*1024)
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
line := scanner.Text()
|
line := scanner.Text()
|
||||||
if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil {
|
if _, err := fmt.Fprintf(w.outFile, " %s\n", line); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOWrite, "failed to write YAML line").WithFilePath(path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOWrite,
|
||||||
|
"failed to write YAML line",
|
||||||
|
).WithFilePath(path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := scanner.Err(); err != nil {
|
if err := scanner.Err(); err != nil {
|
||||||
return utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIORead, "failed to scan YAML content").WithFilePath(path)
|
return gibidiutils.WrapError(
|
||||||
|
err, gibidiutils.ErrorTypeIO, gibidiutils.CodeIOFileRead,
|
||||||
|
"failed to scan YAML content",
|
||||||
|
).WithFilePath(path)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// closeReader safely closes a reader if it implements io.Closer.
|
|
||||||
func (w *YAMLWriter) closeReader(reader io.Reader, path string) {
|
|
||||||
if closer, ok := reader.(io.Closer); ok {
|
|
||||||
if err := closer.Close(); err != nil {
|
|
||||||
utils.LogError(
|
|
||||||
"Failed to close file reader",
|
|
||||||
utils.WrapError(err, utils.ErrorTypeIO, utils.CodeIOClose, "failed to close file reader").WithFilePath(path),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// yamlQuoteString quotes a string for YAML output if needed.
|
|
||||||
func yamlQuoteString(s string) string {
|
|
||||||
if s == "" {
|
|
||||||
return `""`
|
|
||||||
}
|
|
||||||
// Simple YAML quoting - use double quotes if string contains special characters
|
|
||||||
if strings.ContainsAny(s, "\n\r\t:\"'\\") {
|
|
||||||
return fmt.Sprintf(`"%s"`, strings.ReplaceAll(s, `"`, `\"`))
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// startYAMLWriter handles YAML format output with streaming support.
|
// startYAMLWriter handles YAML format output with streaming support.
|
||||||
func startYAMLWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
|
func startYAMLWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<- struct{}, prefix, suffix string) {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
@@ -130,19 +290,19 @@ func startYAMLWriter(outFile *os.File, writeCh <-chan WriteRequest, done chan<-
|
|||||||
|
|
||||||
// Start writing
|
// Start writing
|
||||||
if err := writer.Start(prefix, suffix); err != nil {
|
if err := writer.Start(prefix, suffix); err != nil {
|
||||||
utils.LogError("Failed to write YAML header", err)
|
gibidiutils.LogError("Failed to write YAML header", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process files
|
// Process files
|
||||||
for req := range writeCh {
|
for req := range writeCh {
|
||||||
if err := writer.WriteFile(req); err != nil {
|
if err := writer.WriteFile(req); err != nil {
|
||||||
utils.LogError("Failed to write YAML file", err)
|
gibidiutils.LogError("Failed to write YAML file", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close writer
|
// Close writer
|
||||||
if err := writer.Close(); err != nil {
|
if err := writer.Close(); err != nil {
|
||||||
utils.LogError("Failed to write YAML end", err)
|
gibidiutils.LogError("Failed to write YAML end", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
// Package utils provides common utility functions.
|
// Package gibidiutils provides common utility functions for gibidify.
|
||||||
package utils
|
package gibidiutils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
@@ -47,6 +50,11 @@ func (e ErrorType) String() string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Error formatting templates.
|
||||||
|
const (
|
||||||
|
errorFormatWithCause = "%s: %v"
|
||||||
|
)
|
||||||
|
|
||||||
// StructuredError represents a structured error with type, code, and context.
|
// StructuredError represents a structured error with type, code, and context.
|
||||||
type StructuredError struct {
|
type StructuredError struct {
|
||||||
Type ErrorType
|
Type ErrorType
|
||||||
@@ -60,10 +68,25 @@ type StructuredError struct {
|
|||||||
|
|
||||||
// Error implements the error interface.
|
// Error implements the error interface.
|
||||||
func (e *StructuredError) Error() string {
|
func (e *StructuredError) Error() string {
|
||||||
if e.Cause != nil {
|
base := fmt.Sprintf("%s [%s]: %s", e.Type, e.Code, e.Message)
|
||||||
return fmt.Sprintf("%s [%s]: %s: %v", e.Type, e.Code, e.Message, e.Cause)
|
if len(e.Context) > 0 {
|
||||||
|
// Sort keys for deterministic output
|
||||||
|
keys := make([]string, 0, len(e.Context))
|
||||||
|
for k := range e.Context {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
ctxPairs := make([]string, 0, len(e.Context))
|
||||||
|
for _, k := range keys {
|
||||||
|
ctxPairs = append(ctxPairs, fmt.Sprintf("%s=%v", k, e.Context[k]))
|
||||||
|
}
|
||||||
|
base = fmt.Sprintf("%s | context: %s", base, strings.Join(ctxPairs, ", "))
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s [%s]: %s", e.Type, e.Code, e.Message)
|
if e.Cause != nil {
|
||||||
|
return fmt.Sprintf(errorFormatWithCause, base, e.Cause)
|
||||||
|
}
|
||||||
|
return base
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unwrap returns the underlying cause error.
|
// Unwrap returns the underlying cause error.
|
||||||
@@ -93,7 +116,11 @@ func (e *StructuredError) WithLine(line int) *StructuredError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewStructuredError creates a new structured error.
|
// NewStructuredError creates a new structured error.
|
||||||
func NewStructuredError(errorType ErrorType, code, message, filePath string, context map[string]interface{}) *StructuredError {
|
func NewStructuredError(
|
||||||
|
errorType ErrorType,
|
||||||
|
code, message, filePath string,
|
||||||
|
context map[string]any,
|
||||||
|
) *StructuredError {
|
||||||
return &StructuredError{
|
return &StructuredError{
|
||||||
Type: errorType,
|
Type: errorType,
|
||||||
Code: code,
|
Code: code,
|
||||||
@@ -135,34 +162,40 @@ func WrapErrorf(err error, errorType ErrorType, code, format string, args ...any
|
|||||||
// Common error codes for each type
|
// Common error codes for each type
|
||||||
const (
|
const (
|
||||||
// CLI Error Codes
|
// CLI Error Codes
|
||||||
|
|
||||||
CodeCLIMissingSource = "MISSING_SOURCE"
|
CodeCLIMissingSource = "MISSING_SOURCE"
|
||||||
CodeCLIInvalidArgs = "INVALID_ARGS"
|
CodeCLIInvalidArgs = "INVALID_ARGS"
|
||||||
|
|
||||||
// FileSystem Error Codes
|
// FileSystem Error Codes
|
||||||
|
|
||||||
CodeFSPathResolution = "PATH_RESOLUTION"
|
CodeFSPathResolution = "PATH_RESOLUTION"
|
||||||
CodeFSPermission = "PERMISSION_DENIED"
|
CodeFSPermission = "PERMISSION_DENIED"
|
||||||
CodeFSNotFound = "NOT_FOUND"
|
CodeFSNotFound = "NOT_FOUND"
|
||||||
CodeFSAccess = "ACCESS_DENIED"
|
CodeFSAccess = "ACCESS_DENIED"
|
||||||
|
|
||||||
// Processing Error Codes
|
// Processing Error Codes
|
||||||
|
|
||||||
CodeProcessingFileRead = "FILE_READ"
|
CodeProcessingFileRead = "FILE_READ"
|
||||||
CodeProcessingCollection = "COLLECTION"
|
CodeProcessingCollection = "COLLECTION"
|
||||||
CodeProcessingTraversal = "TRAVERSAL"
|
CodeProcessingTraversal = "TRAVERSAL"
|
||||||
CodeProcessingEncode = "ENCODE"
|
CodeProcessingEncode = "ENCODE"
|
||||||
|
|
||||||
// Configuration Error Codes
|
// Configuration Error Codes
|
||||||
|
|
||||||
CodeConfigValidation = "VALIDATION"
|
CodeConfigValidation = "VALIDATION"
|
||||||
CodeConfigMissing = "MISSING"
|
CodeConfigMissing = "MISSING"
|
||||||
|
|
||||||
// IO Error Codes
|
// IO Error Codes
|
||||||
|
|
||||||
CodeIOFileCreate = "FILE_CREATE"
|
CodeIOFileCreate = "FILE_CREATE"
|
||||||
CodeIOFileWrite = "FILE_WRITE"
|
CodeIOFileWrite = "FILE_WRITE"
|
||||||
CodeIOEncoding = "ENCODING"
|
CodeIOEncoding = "ENCODING"
|
||||||
CodeIOWrite = "WRITE"
|
CodeIOWrite = "WRITE"
|
||||||
CodeIORead = "READ"
|
CodeIOFileRead = "FILE_READ"
|
||||||
CodeIOClose = "CLOSE"
|
CodeIOClose = "CLOSE"
|
||||||
|
|
||||||
// Validation Error Codes
|
// Validation Error Codes
|
||||||
|
|
||||||
CodeValidationFormat = "FORMAT"
|
CodeValidationFormat = "FORMAT"
|
||||||
CodeValidationFileType = "FILE_TYPE"
|
CodeValidationFileType = "FILE_TYPE"
|
||||||
CodeValidationSize = "SIZE_LIMIT"
|
CodeValidationSize = "SIZE_LIMIT"
|
||||||
@@ -170,6 +203,7 @@ const (
|
|||||||
CodeValidationPath = "PATH_TRAVERSAL"
|
CodeValidationPath = "PATH_TRAVERSAL"
|
||||||
|
|
||||||
// Resource Limit Error Codes
|
// Resource Limit Error Codes
|
||||||
|
|
||||||
CodeResourceLimitFiles = "FILE_COUNT_LIMIT"
|
CodeResourceLimitFiles = "FILE_COUNT_LIMIT"
|
||||||
CodeResourceLimitTotalSize = "TOTAL_SIZE_LIMIT"
|
CodeResourceLimitTotalSize = "TOTAL_SIZE_LIMIT"
|
||||||
CodeResourceLimitTimeout = "TIMEOUT"
|
CodeResourceLimitTimeout = "TIMEOUT"
|
||||||
@@ -180,9 +214,16 @@ const (
|
|||||||
|
|
||||||
// Predefined error constructors for common error scenarios
|
// Predefined error constructors for common error scenarios
|
||||||
|
|
||||||
// NewCLIMissingSourceError creates a CLI error for missing source argument.
|
// NewMissingSourceError creates a CLI error for missing source argument.
|
||||||
func NewCLIMissingSourceError() *StructuredError {
|
func NewMissingSourceError() *StructuredError {
|
||||||
return NewStructuredError(ErrorTypeCLI, CodeCLIMissingSource, "usage: gibidify -source <source_directory> [--destination <output_file>] [--format=json|yaml|markdown]", "", nil)
|
return NewStructuredError(
|
||||||
|
ErrorTypeCLI,
|
||||||
|
CodeCLIMissingSource,
|
||||||
|
"usage: gibidify -source <source_directory> "+
|
||||||
|
"[--destination <output_file>] [--format=json|yaml|markdown]",
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFileSystemError creates a file system error.
|
// NewFileSystemError creates a file system error.
|
||||||
@@ -217,16 +258,18 @@ func LogError(operation string, err error, args ...any) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check if it's a structured error and log with additional context
|
// Check if it's a structured error and log with additional context
|
||||||
if structErr, ok := err.(*StructuredError); ok {
|
var structErr *StructuredError
|
||||||
|
if errors.As(err, &structErr) {
|
||||||
logrus.WithFields(logrus.Fields{
|
logrus.WithFields(logrus.Fields{
|
||||||
"error_type": structErr.Type.String(),
|
"error_type": structErr.Type.String(),
|
||||||
"error_code": structErr.Code,
|
"error_code": structErr.Code,
|
||||||
"context": structErr.Context,
|
"context": structErr.Context,
|
||||||
"file_path": structErr.FilePath,
|
"file_path": structErr.FilePath,
|
||||||
"line": structErr.Line,
|
"line": structErr.Line,
|
||||||
}).Errorf("%s: %v", msg, err)
|
}).Errorf(errorFormatWithCause, msg, err)
|
||||||
} else {
|
} else {
|
||||||
logrus.Errorf("%s: %v", msg, err)
|
// Log regular errors without structured fields
|
||||||
|
logrus.Errorf(errorFormatWithCause, msg, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
367
gibidiutils/errors_additional_test.go
Normal file
367
gibidiutils/errors_additional_test.go
Normal file
@@ -0,0 +1,367 @@
|
|||||||
|
package gibidiutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestErrorTypeString(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
errType ErrorType
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "CLI error type",
|
||||||
|
errType: ErrorTypeCLI,
|
||||||
|
expected: "CLI",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "FileSystem error type",
|
||||||
|
errType: ErrorTypeFileSystem,
|
||||||
|
expected: "FileSystem",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Processing error type",
|
||||||
|
errType: ErrorTypeProcessing,
|
||||||
|
expected: "Processing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Configuration error type",
|
||||||
|
errType: ErrorTypeConfiguration,
|
||||||
|
expected: "Configuration",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IO error type",
|
||||||
|
errType: ErrorTypeIO,
|
||||||
|
expected: "IO",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Validation error type",
|
||||||
|
errType: ErrorTypeValidation,
|
||||||
|
expected: "Validation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Unknown error type",
|
||||||
|
errType: ErrorTypeUnknown,
|
||||||
|
expected: "Unknown",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid error type",
|
||||||
|
errType: ErrorType(999),
|
||||||
|
expected: "Unknown",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := tt.errType.String()
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStructuredErrorMethods(t *testing.T) {
|
||||||
|
t.Run("Error method", func(t *testing.T) {
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeValidation,
|
||||||
|
Code: CodeValidationRequired,
|
||||||
|
Message: "field is required",
|
||||||
|
}
|
||||||
|
expected := "Validation [REQUIRED]: field is required"
|
||||||
|
assert.Equal(t, expected, err.Error())
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Error method with context", func(t *testing.T) {
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeFileSystem,
|
||||||
|
Code: CodeFSNotFound,
|
||||||
|
Message: testErrFileNotFound,
|
||||||
|
Context: map[string]interface{}{
|
||||||
|
"path": "/test/file.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
errStr := err.Error()
|
||||||
|
assert.Contains(t, errStr, "FileSystem")
|
||||||
|
assert.Contains(t, errStr, "NOT_FOUND")
|
||||||
|
assert.Contains(t, errStr, testErrFileNotFound)
|
||||||
|
assert.Contains(t, errStr, "/test/file.txt")
|
||||||
|
assert.Contains(t, errStr, "path")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Unwrap method", func(t *testing.T) {
|
||||||
|
innerErr := errors.New("inner error")
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeIO,
|
||||||
|
Code: CodeIOFileWrite,
|
||||||
|
Message: testErrWriteFailed,
|
||||||
|
Cause: innerErr,
|
||||||
|
}
|
||||||
|
assert.Equal(t, innerErr, err.Unwrap())
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Unwrap with nil cause", func(t *testing.T) {
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeIO,
|
||||||
|
Code: CodeIOFileWrite,
|
||||||
|
Message: testErrWriteFailed,
|
||||||
|
}
|
||||||
|
assert.Nil(t, err.Unwrap())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithContextMethods(t *testing.T) {
|
||||||
|
t.Run("WithContext", func(t *testing.T) {
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeValidation,
|
||||||
|
Code: CodeValidationFormat,
|
||||||
|
Message: testErrInvalidFormat,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = err.WithContext("format", "xml")
|
||||||
|
err = err.WithContext("expected", "json")
|
||||||
|
|
||||||
|
assert.NotNil(t, err.Context)
|
||||||
|
assert.Equal(t, "xml", err.Context["format"])
|
||||||
|
assert.Equal(t, "json", err.Context["expected"])
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("WithFilePath", func(t *testing.T) {
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeFileSystem,
|
||||||
|
Code: CodeFSPermission,
|
||||||
|
Message: "permission denied",
|
||||||
|
}
|
||||||
|
|
||||||
|
err = err.WithFilePath("/etc/passwd")
|
||||||
|
|
||||||
|
assert.Equal(t, "/etc/passwd", err.FilePath)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("WithLine", func(t *testing.T) {
|
||||||
|
err := &StructuredError{
|
||||||
|
Type: ErrorTypeProcessing,
|
||||||
|
Code: CodeProcessingFileRead,
|
||||||
|
Message: "read error",
|
||||||
|
}
|
||||||
|
|
||||||
|
err = err.WithLine(42)
|
||||||
|
|
||||||
|
assert.Equal(t, 42, err.Line)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewStructuredError(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
errType ErrorType
|
||||||
|
code string
|
||||||
|
message string
|
||||||
|
filePath string
|
||||||
|
context map[string]interface{}
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "basic error",
|
||||||
|
errType: ErrorTypeValidation,
|
||||||
|
code: CodeValidationRequired,
|
||||||
|
message: "field is required",
|
||||||
|
filePath: "",
|
||||||
|
context: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "error with file path",
|
||||||
|
errType: ErrorTypeFileSystem,
|
||||||
|
code: CodeFSNotFound,
|
||||||
|
message: testErrFileNotFound,
|
||||||
|
filePath: "/test/missing.txt",
|
||||||
|
context: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "error with context",
|
||||||
|
errType: ErrorTypeIO,
|
||||||
|
code: CodeIOFileWrite,
|
||||||
|
message: testErrWriteFailed,
|
||||||
|
context: map[string]interface{}{
|
||||||
|
"size": 1024,
|
||||||
|
"error": "disk full",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := NewStructuredError(tt.errType, tt.code, tt.message, tt.filePath, tt.context)
|
||||||
|
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, tt.errType, err.Type)
|
||||||
|
assert.Equal(t, tt.code, err.Code)
|
||||||
|
assert.Equal(t, tt.message, err.Message)
|
||||||
|
assert.Equal(t, tt.filePath, err.FilePath)
|
||||||
|
assert.Equal(t, tt.context, err.Context)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewStructuredErrorf(t *testing.T) {
|
||||||
|
err := NewStructuredErrorf(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationSize,
|
||||||
|
"file size %d exceeds limit %d",
|
||||||
|
2048, 1024,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, ErrorTypeValidation, err.Type)
|
||||||
|
assert.Equal(t, CodeValidationSize, err.Code)
|
||||||
|
assert.Equal(t, "file size 2048 exceeds limit 1024", err.Message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWrapError(t *testing.T) {
|
||||||
|
innerErr := errors.New("original error")
|
||||||
|
wrappedErr := WrapError(
|
||||||
|
innerErr,
|
||||||
|
ErrorTypeProcessing,
|
||||||
|
CodeProcessingFileRead,
|
||||||
|
"failed to process file",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.NotNil(t, wrappedErr)
|
||||||
|
assert.Equal(t, ErrorTypeProcessing, wrappedErr.Type)
|
||||||
|
assert.Equal(t, CodeProcessingFileRead, wrappedErr.Code)
|
||||||
|
assert.Equal(t, "failed to process file", wrappedErr.Message)
|
||||||
|
assert.Equal(t, innerErr, wrappedErr.Cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWrapErrorf(t *testing.T) {
|
||||||
|
innerErr := errors.New("original error")
|
||||||
|
wrappedErr := WrapErrorf(
|
||||||
|
innerErr,
|
||||||
|
ErrorTypeIO,
|
||||||
|
CodeIOFileCreate,
|
||||||
|
"failed to create %s in %s",
|
||||||
|
"output.txt", "/tmp",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.NotNil(t, wrappedErr)
|
||||||
|
assert.Equal(t, ErrorTypeIO, wrappedErr.Type)
|
||||||
|
assert.Equal(t, CodeIOFileCreate, wrappedErr.Code)
|
||||||
|
assert.Equal(t, "failed to create output.txt in /tmp", wrappedErr.Message)
|
||||||
|
assert.Equal(t, innerErr, wrappedErr.Cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecificErrorConstructors(t *testing.T) {
|
||||||
|
t.Run("NewMissingSourceError", func(t *testing.T) {
|
||||||
|
err := NewMissingSourceError()
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, ErrorTypeCLI, err.Type)
|
||||||
|
assert.Equal(t, CodeCLIMissingSource, err.Code)
|
||||||
|
assert.Contains(t, err.Message, "source")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("NewFileSystemError", func(t *testing.T) {
|
||||||
|
err := NewFileSystemError(CodeFSPermission, "access denied")
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, ErrorTypeFileSystem, err.Type)
|
||||||
|
assert.Equal(t, CodeFSPermission, err.Code)
|
||||||
|
assert.Equal(t, "access denied", err.Message)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("NewProcessingError", func(t *testing.T) {
|
||||||
|
err := NewProcessingError(CodeProcessingCollection, "collection failed")
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, ErrorTypeProcessing, err.Type)
|
||||||
|
assert.Equal(t, CodeProcessingCollection, err.Code)
|
||||||
|
assert.Equal(t, "collection failed", err.Message)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("NewIOError", func(t *testing.T) {
|
||||||
|
err := NewIOError(CodeIOFileWrite, testErrWriteFailed)
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, ErrorTypeIO, err.Type)
|
||||||
|
assert.Equal(t, CodeIOFileWrite, err.Code)
|
||||||
|
assert.Equal(t, testErrWriteFailed, err.Message)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("NewValidationError", func(t *testing.T) {
|
||||||
|
err := NewValidationError(CodeValidationFormat, testErrInvalidFormat)
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Equal(t, ErrorTypeValidation, err.Type)
|
||||||
|
assert.Equal(t, CodeValidationFormat, err.Code)
|
||||||
|
assert.Equal(t, testErrInvalidFormat, err.Message)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogErrorf is already covered in errors_test.go
|
||||||
|
|
||||||
|
func TestStructuredErrorChaining(t *testing.T) {
|
||||||
|
// Test method chaining
|
||||||
|
err := NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSNotFound,
|
||||||
|
testErrFileNotFound,
|
||||||
|
"",
|
||||||
|
nil,
|
||||||
|
).WithFilePath("/test.txt").WithLine(10).WithContext("operation", "read")
|
||||||
|
|
||||||
|
assert.Equal(t, "/test.txt", err.FilePath)
|
||||||
|
assert.Equal(t, 10, err.Line)
|
||||||
|
assert.Equal(t, "read", err.Context["operation"])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestErrorCodes(t *testing.T) {
|
||||||
|
// Test that all error codes are defined
|
||||||
|
codes := []string{
|
||||||
|
CodeCLIMissingSource,
|
||||||
|
CodeCLIInvalidArgs,
|
||||||
|
CodeFSPathResolution,
|
||||||
|
CodeFSPermission,
|
||||||
|
CodeFSNotFound,
|
||||||
|
CodeFSAccess,
|
||||||
|
CodeProcessingFileRead,
|
||||||
|
CodeProcessingCollection,
|
||||||
|
CodeProcessingTraversal,
|
||||||
|
CodeProcessingEncode,
|
||||||
|
CodeConfigValidation,
|
||||||
|
CodeConfigMissing,
|
||||||
|
CodeIOFileCreate,
|
||||||
|
CodeIOFileWrite,
|
||||||
|
CodeIOEncoding,
|
||||||
|
CodeIOWrite,
|
||||||
|
CodeIOFileRead,
|
||||||
|
CodeIOClose,
|
||||||
|
CodeValidationRequired,
|
||||||
|
CodeValidationFormat,
|
||||||
|
CodeValidationSize,
|
||||||
|
CodeValidationPath,
|
||||||
|
CodeResourceLimitFiles,
|
||||||
|
CodeResourceLimitTotalSize,
|
||||||
|
CodeResourceLimitMemory,
|
||||||
|
CodeResourceLimitTimeout,
|
||||||
|
}
|
||||||
|
|
||||||
|
// All codes should be non-empty strings
|
||||||
|
for _, code := range codes {
|
||||||
|
assert.NotEmpty(t, code, "Error code should not be empty")
|
||||||
|
assert.NotEqual(t, "", code, "Error code should be defined")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestErrorUnwrapChain(t *testing.T) {
|
||||||
|
// Test unwrapping through multiple levels
|
||||||
|
innermost := errors.New("innermost error")
|
||||||
|
middle := WrapError(innermost, ErrorTypeIO, CodeIOFileRead, "read failed")
|
||||||
|
outer := WrapError(middle, ErrorTypeProcessing, CodeProcessingFileRead, "processing failed")
|
||||||
|
|
||||||
|
// Test unwrapping
|
||||||
|
assert.Equal(t, middle, outer.Unwrap())
|
||||||
|
assert.Equal(t, innermost, middle.Unwrap())
|
||||||
|
|
||||||
|
// innermost is a plain error, doesn't have Unwrap() method
|
||||||
|
// No need to test it
|
||||||
|
|
||||||
|
// Test error chain messages
|
||||||
|
assert.Contains(t, outer.Error(), "Processing")
|
||||||
|
assert.Contains(t, middle.Error(), "IO")
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
package utils
|
// Package gibidiutils provides common utility functions for gibidify.
|
||||||
|
package gibidiutils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -175,7 +176,7 @@ func TestLogErrorf(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLogErrorConcurrency(t *testing.T) {
|
func TestLogErrorConcurrency(_ *testing.T) {
|
||||||
// Test that LogError is safe for concurrent use
|
// Test that LogError is safe for concurrent use
|
||||||
done := make(chan bool)
|
done := make(chan bool)
|
||||||
for i := 0; i < 10; i++ {
|
for i := 0; i < 10; i++ {
|
||||||
@@ -191,7 +192,7 @@ func TestLogErrorConcurrency(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLogErrorfConcurrency(t *testing.T) {
|
func TestLogErrorfConcurrency(_ *testing.T) {
|
||||||
// Test that LogErrorf is safe for concurrent use
|
// Test that LogErrorf is safe for concurrent use
|
||||||
done := make(chan bool)
|
done := make(chan bool)
|
||||||
for i := 0; i < 10; i++ {
|
for i := 0; i < 10; i++ {
|
||||||
10
gibidiutils/icons.go
Normal file
10
gibidiutils/icons.go
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
package gibidiutils
|
||||||
|
|
||||||
|
// Unicode icons and symbols for CLI UI and test output.
|
||||||
|
const (
|
||||||
|
IconSuccess = "✓" // U+2713
|
||||||
|
IconError = "✗" // U+2717
|
||||||
|
IconWarning = "⚠" // U+26A0
|
||||||
|
IconBullet = "•" // U+2022
|
||||||
|
IconInfo = "ℹ️" // U+2139 FE0F
|
||||||
|
)
|
||||||
311
gibidiutils/paths.go
Normal file
311
gibidiutils/paths.go
Normal file
@@ -0,0 +1,311 @@
|
|||||||
|
// Package gibidiutils provides common utility functions for gibidify.
|
||||||
|
package gibidiutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EscapeForMarkdown sanitizes a string for safe use in Markdown code-fence and header lines.
|
||||||
|
// It replaces backticks with backslash-escaped backticks and removes/collapses newlines.
|
||||||
|
func EscapeForMarkdown(s string) string {
|
||||||
|
// Escape backticks
|
||||||
|
safe := strings.ReplaceAll(s, "`", "\\`")
|
||||||
|
// Remove newlines (collapse to space)
|
||||||
|
safe = strings.ReplaceAll(safe, "\n", " ")
|
||||||
|
safe = strings.ReplaceAll(safe, "\r", " ")
|
||||||
|
return safe
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAbsolutePath returns the absolute path for the given path.
|
||||||
|
// It wraps filepath.Abs with consistent error handling.
|
||||||
|
func GetAbsolutePath(path string) (string, error) {
|
||||||
|
abs, err := filepath.Abs(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to get absolute path for %s: %w", path, err)
|
||||||
|
}
|
||||||
|
return abs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBaseName returns the base name for the given path, handling special cases.
|
||||||
|
func GetBaseName(absPath string) string {
|
||||||
|
baseName := filepath.Base(absPath)
|
||||||
|
if baseName == "." || baseName == "" {
|
||||||
|
return "output"
|
||||||
|
}
|
||||||
|
return baseName
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkPathTraversal checks for path traversal patterns and returns an error if found.
|
||||||
|
func checkPathTraversal(path, context string) error {
|
||||||
|
// Normalize separators without cleaning (to preserve ..)
|
||||||
|
normalized := filepath.ToSlash(path)
|
||||||
|
|
||||||
|
// Split into components
|
||||||
|
components := strings.Split(normalized, "/")
|
||||||
|
|
||||||
|
// Check each component for exact ".." match
|
||||||
|
for _, component := range components {
|
||||||
|
if component == ".." {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
fmt.Sprintf("path traversal attempt detected in %s", context),
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"original_path": path,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanAndResolveAbsPath cleans a path and resolves it to an absolute path.
|
||||||
|
func cleanAndResolveAbsPath(path, context string) (string, error) {
|
||||||
|
cleaned := filepath.Clean(path)
|
||||||
|
abs, err := filepath.Abs(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
return "", NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSPathResolution,
|
||||||
|
fmt.Sprintf("cannot resolve %s", context),
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return abs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// evalSymlinksOrStructuredError wraps filepath.EvalSymlinks with structured error handling.
|
||||||
|
func evalSymlinksOrStructuredError(path, context, original string) (string, error) {
|
||||||
|
eval, err := filepath.EvalSymlinks(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
fmt.Sprintf("cannot resolve symlinks for %s", context),
|
||||||
|
original,
|
||||||
|
map[string]interface{}{
|
||||||
|
"resolved_path": path,
|
||||||
|
"context": context,
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return eval, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateWorkingDirectoryBoundary checks if the given absolute path escapes the working directory.
|
||||||
|
func validateWorkingDirectoryBoundary(abs, path string) error {
|
||||||
|
cwd, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSPathResolution,
|
||||||
|
"cannot get current working directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
cwdAbs, err := filepath.Abs(cwd)
|
||||||
|
if err != nil {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSPathResolution,
|
||||||
|
"cannot resolve current working directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
absEval, err := evalSymlinksOrStructuredError(abs, "source path", path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cwdEval, err := evalSymlinksOrStructuredError(cwdAbs, "working directory", path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(cwdEval, absEval)
|
||||||
|
if err != nil {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
"cannot determine relative path",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"resolved_path": absEval,
|
||||||
|
"working_dir": cwdEval,
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
"source path attempts to access directories outside current working directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"resolved_path": absEval,
|
||||||
|
"working_dir": cwdEval,
|
||||||
|
"relative_path": rel,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateSourcePath validates a source directory path for security.
|
||||||
|
// It ensures the path exists, is a directory, and doesn't contain path traversal attempts.
|
||||||
|
//
|
||||||
|
//revive:disable-next-line:function-length
|
||||||
|
func ValidateSourcePath(path string) error {
|
||||||
|
if path == "" {
|
||||||
|
return NewValidationError(CodeValidationRequired, "source path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for path traversal patterns before cleaning
|
||||||
|
if err := checkPathTraversal(path, "source path"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean and get absolute path
|
||||||
|
abs, err := cleanAndResolveAbsPath(path, "source path")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cleaned := filepath.Clean(path)
|
||||||
|
|
||||||
|
// Ensure the resolved path is within or below the current working directory for relative paths
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
if err := validateWorkingDirectoryBoundary(abs, path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if path exists and is a directory
|
||||||
|
info, err := os.Stat(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return NewFileSystemError(CodeFSNotFound, "source directory does not exist").WithFilePath(path)
|
||||||
|
}
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSAccess,
|
||||||
|
"cannot access source directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !info.IsDir() {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
"source path must be a directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"is_file": true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateDestinationPath validates a destination file path for security.
|
||||||
|
// It ensures the path doesn't contain path traversal attempts and the parent directory exists.
|
||||||
|
func ValidateDestinationPath(path string) error {
|
||||||
|
if path == "" {
|
||||||
|
return NewValidationError(CodeValidationRequired, "destination path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for path traversal patterns before cleaning
|
||||||
|
if err := checkPathTraversal(path, "destination path"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get absolute path to ensure it's not trying to escape current working directory
|
||||||
|
abs, err := cleanAndResolveAbsPath(path, "destination path")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the destination is not a directory
|
||||||
|
if info, err := os.Stat(abs); err == nil && info.IsDir() {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
"destination cannot be a directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"is_directory": true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if parent directory exists and is writable
|
||||||
|
parentDir := filepath.Dir(abs)
|
||||||
|
if parentInfo, err := os.Stat(parentDir); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSNotFound,
|
||||||
|
"destination parent directory does not exist",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"parent_dir": parentDir,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeFileSystem,
|
||||||
|
CodeFSAccess,
|
||||||
|
"cannot access destination parent directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"parent_dir": parentDir,
|
||||||
|
"error": err.Error(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
} else if !parentInfo.IsDir() {
|
||||||
|
return NewStructuredError(
|
||||||
|
ErrorTypeValidation,
|
||||||
|
CodeValidationPath,
|
||||||
|
"destination parent is not a directory",
|
||||||
|
path,
|
||||||
|
map[string]interface{}{
|
||||||
|
"parent_dir": parentDir,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateConfigPath validates a configuration file path for security.
|
||||||
|
// It ensures the path doesn't contain path traversal attempts.
|
||||||
|
func ValidateConfigPath(path string) error {
|
||||||
|
if path == "" {
|
||||||
|
return nil // Empty path is allowed for config
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for path traversal patterns before cleaning
|
||||||
|
return checkPathTraversal(path, "config path")
|
||||||
|
}
|
||||||
368
gibidiutils/paths_additional_test.go
Normal file
368
gibidiutils/paths_additional_test.go
Normal file
@@ -0,0 +1,368 @@
|
|||||||
|
package gibidiutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetBaseName(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
absPath string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "normal path",
|
||||||
|
absPath: "/home/user/project",
|
||||||
|
expected: "project",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path with trailing slash",
|
||||||
|
absPath: "/home/user/project/",
|
||||||
|
expected: "project",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "root path",
|
||||||
|
absPath: "/",
|
||||||
|
expected: "/",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "current directory",
|
||||||
|
absPath: ".",
|
||||||
|
expected: "output",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testEmptyPath,
|
||||||
|
absPath: "",
|
||||||
|
expected: "output",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file path",
|
||||||
|
absPath: "/home/user/file.txt",
|
||||||
|
expected: "file.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := GetBaseName(tt.absPath)
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateSourcePath(t *testing.T) {
|
||||||
|
// Create a temp directory for testing
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
tempFile := filepath.Join(tempDir, "test.txt")
|
||||||
|
require.NoError(t, os.WriteFile(tempFile, []byte("test"), 0o600))
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
expectedError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testEmptyPath,
|
||||||
|
path: "",
|
||||||
|
expectedError: "source path is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testPathTraversalAttempt,
|
||||||
|
path: "../../../etc/passwd",
|
||||||
|
expectedError: testPathTraversalDetected,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path with double dots",
|
||||||
|
path: "/home/../etc/passwd",
|
||||||
|
expectedError: testPathTraversalDetected,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-existent path",
|
||||||
|
path: "/definitely/does/not/exist",
|
||||||
|
expectedError: "does not exist",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file instead of directory",
|
||||||
|
path: tempFile,
|
||||||
|
expectedError: "must be a directory",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid directory",
|
||||||
|
path: tempDir,
|
||||||
|
expectedError: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid relative path",
|
||||||
|
path: ".",
|
||||||
|
expectedError: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := ValidateSourcePath(tt.path)
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), tt.expectedError)
|
||||||
|
|
||||||
|
// Check if it's a StructuredError
|
||||||
|
var structErr *StructuredError
|
||||||
|
if errors.As(err, &structErr) {
|
||||||
|
assert.NotEmpty(t, structErr.Code)
|
||||||
|
assert.NotEqual(t, ErrorTypeUnknown, structErr.Type)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateDestinationPath(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
expectedError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testEmptyPath,
|
||||||
|
path: "",
|
||||||
|
expectedError: "destination path is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testPathTraversalAttempt,
|
||||||
|
path: "../../etc/passwd",
|
||||||
|
expectedError: testPathTraversalDetected,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "absolute path traversal",
|
||||||
|
path: "/home/../../../etc/passwd",
|
||||||
|
expectedError: testPathTraversalDetected,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid new file",
|
||||||
|
path: filepath.Join(tempDir, "newfile.txt"),
|
||||||
|
expectedError: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid relative path",
|
||||||
|
path: "output.txt",
|
||||||
|
expectedError: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := ValidateDestinationPath(tt.path)
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), tt.expectedError)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateConfigPath(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
validConfig := filepath.Join(tempDir, "config.yaml")
|
||||||
|
require.NoError(t, os.WriteFile(validConfig, []byte("key: value"), 0o600))
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
expectedError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: testEmptyPath,
|
||||||
|
path: "",
|
||||||
|
expectedError: "", // Empty config path is allowed
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: testPathTraversalAttempt,
|
||||||
|
path: "../../../etc/config.yaml",
|
||||||
|
expectedError: testPathTraversalDetected,
|
||||||
|
},
|
||||||
|
// ValidateConfigPath doesn't check if file exists or is regular file
|
||||||
|
// It only checks for path traversal
|
||||||
|
{
|
||||||
|
name: "valid config file",
|
||||||
|
path: validConfig,
|
||||||
|
expectedError: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := ValidateConfigPath(tt.path)
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), tt.expectedError)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGetAbsolutePath is already covered in paths_test.go
|
||||||
|
|
||||||
|
func TestValidationErrorTypes(t *testing.T) {
|
||||||
|
t.Run("source path validation errors", func(t *testing.T) {
|
||||||
|
// Test empty source
|
||||||
|
err := ValidateSourcePath("")
|
||||||
|
assert.Error(t, err)
|
||||||
|
var structErrEmptyPath *StructuredError
|
||||||
|
if errors.As(err, &structErrEmptyPath) {
|
||||||
|
assert.Equal(t, ErrorTypeValidation, structErrEmptyPath.Type)
|
||||||
|
assert.Equal(t, CodeValidationRequired, structErrEmptyPath.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test path traversal
|
||||||
|
err = ValidateSourcePath("../../../etc")
|
||||||
|
assert.Error(t, err)
|
||||||
|
var structErrTraversal *StructuredError
|
||||||
|
if errors.As(err, &structErrTraversal) {
|
||||||
|
assert.Equal(t, ErrorTypeValidation, structErrTraversal.Type)
|
||||||
|
assert.Equal(t, CodeValidationPath, structErrTraversal.Code)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("destination path validation errors", func(t *testing.T) {
|
||||||
|
// Test empty destination
|
||||||
|
err := ValidateDestinationPath("")
|
||||||
|
assert.Error(t, err)
|
||||||
|
var structErrEmptyDest *StructuredError
|
||||||
|
if errors.As(err, &structErrEmptyDest) {
|
||||||
|
assert.Equal(t, ErrorTypeValidation, structErrEmptyDest.Type)
|
||||||
|
assert.Equal(t, CodeValidationRequired, structErrEmptyDest.Code)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("config path validation errors", func(t *testing.T) {
|
||||||
|
// Test path traversal in config
|
||||||
|
err := ValidateConfigPath("../../etc/config.yaml")
|
||||||
|
assert.Error(t, err)
|
||||||
|
var structErrTraversalInConfig *StructuredError
|
||||||
|
if errors.As(err, &structErrTraversalInConfig) {
|
||||||
|
assert.Equal(t, ErrorTypeValidation, structErrTraversalInConfig.Type)
|
||||||
|
assert.Equal(t, CodeValidationPath, structErrTraversalInConfig.Code)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPathSecurityChecks(t *testing.T) {
|
||||||
|
// Test various path traversal attempts
|
||||||
|
traversalPaths := []string{
|
||||||
|
"../etc/passwd",
|
||||||
|
"../../root/.ssh/id_rsa",
|
||||||
|
"/home/../../../etc/shadow",
|
||||||
|
"./../../sensitive/data",
|
||||||
|
"foo/../../../bar",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range traversalPaths {
|
||||||
|
t.Run("source_"+path, func(t *testing.T) {
|
||||||
|
err := ValidateSourcePath(path)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testPathTraversal)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("dest_"+path, func(t *testing.T) {
|
||||||
|
err := ValidateDestinationPath(path)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testPathTraversal)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("config_"+path, func(t *testing.T) {
|
||||||
|
err := ValidateConfigPath(path)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testPathTraversal)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpecialPaths(t *testing.T) {
|
||||||
|
t.Run("GetBaseName with special paths", func(t *testing.T) {
|
||||||
|
specialPaths := map[string]string{
|
||||||
|
"/": "/",
|
||||||
|
"": "output",
|
||||||
|
".": "output",
|
||||||
|
"..": "..",
|
||||||
|
"/.": "output", // filepath.Base("/.") returns "." which matches the output condition
|
||||||
|
"/..": "..",
|
||||||
|
"//": "/",
|
||||||
|
"///": "/",
|
||||||
|
}
|
||||||
|
|
||||||
|
for path, expected := range specialPaths {
|
||||||
|
result := GetBaseName(path)
|
||||||
|
assert.Equal(t, expected, result, "Path: %s", path)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPathNormalization(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
t.Run("source path normalization", func(t *testing.T) {
|
||||||
|
// Create nested directory
|
||||||
|
nestedDir := filepath.Join(tempDir, "a", "b", "c")
|
||||||
|
require.NoError(t, os.MkdirAll(nestedDir, 0o750))
|
||||||
|
|
||||||
|
// Test path with redundant separators
|
||||||
|
redundantPath := tempDir + string(
|
||||||
|
os.PathSeparator,
|
||||||
|
) + string(
|
||||||
|
os.PathSeparator,
|
||||||
|
) + "a" + string(
|
||||||
|
os.PathSeparator,
|
||||||
|
) + "b" + string(
|
||||||
|
os.PathSeparator,
|
||||||
|
) + "c"
|
||||||
|
err := ValidateSourcePath(redundantPath)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPathValidationConcurrency(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
// Test concurrent path validation
|
||||||
|
paths := []string{
|
||||||
|
tempDir,
|
||||||
|
".",
|
||||||
|
"/tmp",
|
||||||
|
}
|
||||||
|
|
||||||
|
errChan := make(chan error, len(paths)*2)
|
||||||
|
|
||||||
|
for _, path := range paths {
|
||||||
|
go func(p string) {
|
||||||
|
errChan <- ValidateSourcePath(p)
|
||||||
|
}(path)
|
||||||
|
|
||||||
|
go func(p string) {
|
||||||
|
errChan <- ValidateDestinationPath(p + "/output.txt")
|
||||||
|
}(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect results
|
||||||
|
for i := 0; i < len(paths)*2; i++ {
|
||||||
|
<-errChan
|
||||||
|
}
|
||||||
|
|
||||||
|
// No assertions needed - test passes if no panic/race
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
package utils
|
// Package gibidiutils provides common utility functions for gibidify.
|
||||||
|
package gibidiutils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
@@ -138,7 +139,7 @@ func TestGetAbsolutePathSpecialCases(t *testing.T) {
|
|||||||
target := filepath.Join(tmpDir, "target")
|
target := filepath.Join(tmpDir, "target")
|
||||||
link := filepath.Join(tmpDir, "link")
|
link := filepath.Join(tmpDir, "link")
|
||||||
|
|
||||||
if err := os.Mkdir(target, 0o755); err != nil {
|
if err := os.Mkdir(target, 0o750); err != nil {
|
||||||
t.Fatalf("Failed to create target directory: %v", err)
|
t.Fatalf("Failed to create target directory: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.Symlink(target, link); err != nil {
|
if err := os.Symlink(target, link); err != nil {
|
||||||
@@ -189,7 +190,10 @@ func TestGetAbsolutePathSpecialCases(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetAbsolutePathConcurrency(t *testing.T) {
|
// TestGetAbsolutePathConcurrency verifies that GetAbsolutePath is safe for concurrent use.
|
||||||
|
// The test intentionally does not use assertions - it will panic if there's a race condition.
|
||||||
|
// Run with -race flag to detect concurrent access issues.
|
||||||
|
func TestGetAbsolutePathConcurrency(_ *testing.T) {
|
||||||
// Test that GetAbsolutePath is safe for concurrent use
|
// Test that GetAbsolutePath is safe for concurrent use
|
||||||
paths := []string{".", "..", "test.go", "subdir/file.txt", "/tmp/test"}
|
paths := []string{".", "..", "test.go", "subdir/file.txt", "/tmp/test"}
|
||||||
done := make(chan bool)
|
done := make(chan bool)
|
||||||
@@ -224,11 +228,9 @@ func TestGetAbsolutePathErrorFormatting(t *testing.T) {
|
|||||||
if !strings.Contains(err.Error(), path) {
|
if !strings.Contains(err.Error(), path) {
|
||||||
t.Errorf("Error message should contain original path: %v", err)
|
t.Errorf("Error message should contain original path: %v", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else if !filepath.IsAbs(got) {
|
||||||
// Normal case - just verify we got a valid absolute path
|
// Normal case - just verify we got a valid absolute path
|
||||||
if !filepath.IsAbs(got) {
|
t.Errorf("Expected absolute path, got: %v", got)
|
||||||
t.Errorf("Expected absolute path, got: %v", got)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
18
gibidiutils/test_constants.go
Normal file
18
gibidiutils/test_constants.go
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
package gibidiutils
|
||||||
|
|
||||||
|
// Test constants to avoid duplication in test files.
|
||||||
|
// These constants are used across multiple test files in the gibidiutils package.
|
||||||
|
const (
|
||||||
|
// Error messages
|
||||||
|
|
||||||
|
testErrFileNotFound = "file not found"
|
||||||
|
testErrWriteFailed = "write failed"
|
||||||
|
testErrInvalidFormat = "invalid format"
|
||||||
|
|
||||||
|
// Path validation messages
|
||||||
|
|
||||||
|
testEmptyPath = "empty path"
|
||||||
|
testPathTraversal = "path traversal"
|
||||||
|
testPathTraversalAttempt = "path traversal attempt"
|
||||||
|
testPathTraversalDetected = "path traversal attempt detected"
|
||||||
|
)
|
||||||
@@ -1,8 +1,10 @@
|
|||||||
package utils
|
// Package gibidiutils provides common utility functions for gibidify.
|
||||||
|
package gibidiutils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
|
"math"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,7 +36,15 @@ func WriteWithErrorWrap(writer io.Writer, content, errorMsg, filePath string) er
|
|||||||
|
|
||||||
// StreamContent provides a common streaming implementation with chunk processing.
|
// StreamContent provides a common streaming implementation with chunk processing.
|
||||||
// This eliminates the similar streaming patterns across JSON and Markdown writers.
|
// This eliminates the similar streaming patterns across JSON and Markdown writers.
|
||||||
func StreamContent(reader io.Reader, writer io.Writer, chunkSize int, filePath string, processChunk func([]byte) []byte) error {
|
//
|
||||||
|
//revive:disable-next-line:cognitive-complexity
|
||||||
|
func StreamContent(
|
||||||
|
reader io.Reader,
|
||||||
|
writer io.Writer,
|
||||||
|
chunkSize int,
|
||||||
|
filePath string,
|
||||||
|
processChunk func([]byte) []byte,
|
||||||
|
) error {
|
||||||
buf := make([]byte, chunkSize)
|
buf := make([]byte, chunkSize)
|
||||||
for {
|
for {
|
||||||
n, err := reader.Read(buf)
|
n, err := reader.Read(buf)
|
||||||
@@ -55,7 +65,7 @@ func StreamContent(reader io.Reader, writer io.Writer, chunkSize int, filePath s
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wrappedErr := WrapError(err, ErrorTypeIO, CodeIORead, "failed to read content chunk")
|
wrappedErr := WrapError(err, ErrorTypeIO, CodeIOFileRead, "failed to read content chunk")
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
wrappedErr = wrappedErr.WithFilePath(filePath)
|
wrappedErr = wrappedErr.WithFilePath(filePath)
|
||||||
}
|
}
|
||||||
@@ -99,13 +109,27 @@ func EscapeForYAML(content string) string {
|
|||||||
return content
|
return content
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SafeUint64ToInt64WithDefault safely converts uint64 to int64, returning a default value if overflow would occur.
|
||||||
|
// When defaultValue is 0 (the safe default), clamps to MaxInt64 on overflow to keep guardrails active.
|
||||||
|
// This prevents overflow from making monitors think memory usage is zero when it's actually maxed out.
|
||||||
|
func SafeUint64ToInt64WithDefault(value uint64, defaultValue int64) int64 {
|
||||||
|
if value > math.MaxInt64 {
|
||||||
|
// When caller uses 0 as "safe" default, clamp to max so overflow still trips guardrails
|
||||||
|
if defaultValue == 0 {
|
||||||
|
return math.MaxInt64
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return int64(value) //#nosec G115 -- Safe: value <= MaxInt64 checked above
|
||||||
|
}
|
||||||
|
|
||||||
// StreamLines provides line-based streaming for YAML content.
|
// StreamLines provides line-based streaming for YAML content.
|
||||||
// This provides an alternative streaming approach for YAML writers.
|
// This provides an alternative streaming approach for YAML writers.
|
||||||
func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProcessor func(string) string) error {
|
func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProcessor func(string) string) error {
|
||||||
// Read all content first (for small files this is fine)
|
// Read all content first (for small files this is fine)
|
||||||
content, err := io.ReadAll(reader)
|
content, err := io.ReadAll(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wrappedErr := WrapError(err, ErrorTypeIO, CodeIORead, "failed to read content for line processing")
|
wrappedErr := WrapError(err, ErrorTypeIO, CodeIOFileRead, "failed to read content for line processing")
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
wrappedErr = wrappedErr.WithFilePath(filePath)
|
wrappedErr = wrappedErr.WithFilePath(filePath)
|
||||||
}
|
}
|
||||||
@@ -119,13 +143,13 @@ func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProces
|
|||||||
if lineProcessor != nil {
|
if lineProcessor != nil {
|
||||||
processedLine = lineProcessor(line)
|
processedLine = lineProcessor(line)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write line with proper line ending (except for last empty line)
|
// Write line with proper line ending (except for last empty line)
|
||||||
lineToWrite := processedLine
|
lineToWrite := processedLine
|
||||||
if i < len(lines)-1 || line != "" {
|
if i < len(lines)-1 || line != "" {
|
||||||
lineToWrite += "\n"
|
lineToWrite += "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, writeErr := writer.Write([]byte(lineToWrite)); writeErr != nil {
|
if _, writeErr := writer.Write([]byte(lineToWrite)); writeErr != nil {
|
||||||
wrappedErr := WrapError(writeErr, ErrorTypeIO, CodeIOWrite, "failed to write processed line")
|
wrappedErr := WrapError(writeErr, ErrorTypeIO, CodeIOWrite, "failed to write processed line")
|
||||||
if filePath != "" {
|
if filePath != "" {
|
||||||
@@ -135,4 +159,4 @@ func StreamLines(reader io.Reader, writer io.Writer, filePath string, lineProces
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
111
gibidiutils/writers_test.go
Normal file
111
gibidiutils/writers_test.go
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
// Package gibidiutils provides common utility functions for gibidify.
|
||||||
|
package gibidiutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSafeUint64ToInt64WithDefault(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value uint64
|
||||||
|
defaultValue int64
|
||||||
|
want int64
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "normal value within range",
|
||||||
|
value: 1000,
|
||||||
|
defaultValue: 0,
|
||||||
|
want: 1000,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero value",
|
||||||
|
value: 0,
|
||||||
|
defaultValue: 0,
|
||||||
|
want: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "max int64 exactly",
|
||||||
|
value: math.MaxInt64,
|
||||||
|
defaultValue: 0,
|
||||||
|
want: math.MaxInt64,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overflow with zero default clamps to max",
|
||||||
|
value: math.MaxInt64 + 1,
|
||||||
|
defaultValue: 0,
|
||||||
|
want: math.MaxInt64,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "large overflow with zero default clamps to max",
|
||||||
|
value: math.MaxUint64,
|
||||||
|
defaultValue: 0,
|
||||||
|
want: math.MaxInt64,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overflow with custom default returns custom",
|
||||||
|
value: math.MaxInt64 + 1,
|
||||||
|
defaultValue: -1,
|
||||||
|
want: -1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overflow with custom positive default",
|
||||||
|
value: math.MaxUint64,
|
||||||
|
defaultValue: 12345,
|
||||||
|
want: 12345,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "large value within range",
|
||||||
|
value: uint64(math.MaxInt64 - 1000),
|
||||||
|
defaultValue: 0,
|
||||||
|
want: math.MaxInt64 - 1000,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := SafeUint64ToInt64WithDefault(tt.value, tt.defaultValue)
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("SafeUint64ToInt64WithDefault(%d, %d) = %d, want %d",
|
||||||
|
tt.value, tt.defaultValue, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSafeUint64ToInt64WithDefaultGuardrailsBehavior(t *testing.T) {
|
||||||
|
// Test that overflow with default=0 returns MaxInt64, not 0
|
||||||
|
// This is critical for back-pressure and resource monitors
|
||||||
|
result := SafeUint64ToInt64WithDefault(math.MaxUint64, 0)
|
||||||
|
if result == 0 {
|
||||||
|
t.Error("Overflow with default=0 returned 0, which would disable guardrails")
|
||||||
|
}
|
||||||
|
if result != math.MaxInt64 {
|
||||||
|
t.Errorf("Overflow with default=0 should clamp to MaxInt64, got %d", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkSafeUint64ToInt64WithDefault benchmarks the conversion function
|
||||||
|
func BenchmarkSafeUint64ToInt64WithDefault(b *testing.B) {
|
||||||
|
b.Run("normal_value", func(b *testing.B) {
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = SafeUint64ToInt64WithDefault(1000, 0)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
b.Run("overflow_zero_default", func(b *testing.B) {
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = SafeUint64ToInt64WithDefault(math.MaxUint64, 0)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
b.Run("overflow_custom_default", func(b *testing.B) {
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = SafeUint64ToInt64WithDefault(math.MaxUint64, -1)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
18
go.mod
18
go.mod
@@ -1,6 +1,6 @@
|
|||||||
module github.com/ivuorinen/gibidify
|
module github.com/ivuorinen/gibidify
|
||||||
|
|
||||||
go 1.24.1
|
go 1.25
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/fatih/color v1.18.0
|
github.com/fatih/color v1.18.0
|
||||||
@@ -8,26 +8,28 @@ require (
|
|||||||
github.com/schollz/progressbar/v3 v3.18.0
|
github.com/schollz/progressbar/v3 v3.18.0
|
||||||
github.com/sirupsen/logrus v1.9.3
|
github.com/sirupsen/logrus v1.9.3
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
|
github.com/stretchr/testify v1.11.1
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
|
||||||
github.com/spf13/afero v1.15.0 // indirect
|
github.com/spf13/afero v1.15.0 // indirect
|
||||||
github.com/spf13/cast v1.10.0 // indirect
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.10 // indirect
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
github.com/subosito/gotenv v1.6.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
golang.org/x/sys v0.31.0 // indirect
|
golang.org/x/sys v0.37.0 // indirect
|
||||||
golang.org/x/term v0.28.0 // indirect
|
golang.org/x/term v0.36.0 // indirect
|
||||||
golang.org/x/text v0.28.0 // indirect
|
golang.org/x/text v0.30.0 // indirect
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
52
go.sum
52
go.sum
@@ -7,12 +7,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
|||||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
|
|
||||||
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
|
|
||||||
github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
@@ -21,17 +17,14 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
|||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
|
|
||||||
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
|
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
@@ -42,58 +35,37 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV
|
|||||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
|
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
|
||||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||||
github.com/sagikazarmark/locafero v0.8.0 h1:mXaMVw7IqxNBxfv3LdWt9MDmcWDQ1fagDH918lOdVaQ=
|
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
|
||||||
github.com/sagikazarmark/locafero v0.8.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk=
|
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
||||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
|
||||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
|
||||||
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
|
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
|
||||||
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
|
||||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
|
||||||
github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA=
|
|
||||||
github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo=
|
|
||||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
|
||||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
|
||||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
|
||||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
|
||||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY=
|
|
||||||
github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
|
|
||||||
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
|
|
||||||
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
|
|
||||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
|
||||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
|
||||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
|
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||||
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
|
golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
|
||||||
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
|
||||||
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
|
||||||
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
|
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
|
||||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
|
||||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
|||||||
9
main.go
9
main.go
@@ -23,12 +23,11 @@ func main() {
|
|||||||
if cli.IsUserError(err) {
|
if cli.IsUserError(err) {
|
||||||
errorFormatter.FormatError(err)
|
errorFormatter.FormatError(err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
} else {
|
|
||||||
// System errors still go to logrus for debugging
|
|
||||||
logrus.Errorf("System error: %v", err)
|
|
||||||
ui.PrintError("An unexpected error occurred. Please check the logs.")
|
|
||||||
os.Exit(2)
|
|
||||||
}
|
}
|
||||||
|
// System errors still go to logrus for debugging
|
||||||
|
logrus.Errorf("System error: %v", err)
|
||||||
|
ui.PrintError("An unexpected error occurred. Please check the logs.")
|
||||||
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
14
main_test.go
14
main_test.go
@@ -7,6 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/ivuorinen/gibidify/config"
|
||||||
"github.com/ivuorinen/gibidify/testutil"
|
"github.com/ivuorinen/gibidify/testutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -14,13 +15,22 @@ const (
|
|||||||
testFileCount = 1000
|
testFileCount = 1000
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TestMain configures test-time flags for packages.
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
// Inform packages that we're running under tests so they can adjust noisy logging.
|
||||||
|
// The config package will suppress the specific info-level message about missing config
|
||||||
|
// while still allowing tests to enable debug/info level logging when needed.
|
||||||
|
config.SetRunningInTest(true)
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
|
||||||
// TestIntegrationFullCLI simulates a full run of the CLI application using adaptive concurrency.
|
// TestIntegrationFullCLI simulates a full run of the CLI application using adaptive concurrency.
|
||||||
func TestIntegrationFullCLI(t *testing.T) {
|
func TestIntegrationFullCLI(t *testing.T) {
|
||||||
srcDir := setupTestFiles(t)
|
srcDir := setupTestFiles(t)
|
||||||
outFilePath := setupOutputFile(t)
|
outFilePath := setupOutputFile(t)
|
||||||
setupCLIArgs(srcDir, outFilePath)
|
setupCLIArgs(srcDir, outFilePath)
|
||||||
|
|
||||||
// Run the application with a background context.
|
// Run the application with the test context.
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
if runErr := run(ctx); runErr != nil {
|
if runErr := run(ctx); runErr != nil {
|
||||||
t.Fatalf("Run failed: %v", runErr)
|
t.Fatalf("Run failed: %v", runErr)
|
||||||
@@ -60,7 +70,7 @@ func setupCLIArgs(srcDir, outFilePath string) {
|
|||||||
// verifyOutput checks that the output file contains expected content.
|
// verifyOutput checks that the output file contains expected content.
|
||||||
func verifyOutput(t *testing.T, outFilePath string) {
|
func verifyOutput(t *testing.T, outFilePath string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(outFilePath)
|
data, err := os.ReadFile(outFilePath) // #nosec G304 - test file path is controlled
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to read output file: %v", err)
|
t.Fatalf("Failed to read output file: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
58
revive.toml
Normal file
58
revive.toml
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
# revive configuration for gibidify
|
||||||
|
# See https://revive.run/ for more information
|
||||||
|
|
||||||
|
# Global settings
|
||||||
|
ignoreGeneratedHeader = false
|
||||||
|
severity = "warning"
|
||||||
|
confidence = 0.8
|
||||||
|
errorCode = 1
|
||||||
|
warningCode = 0
|
||||||
|
|
||||||
|
# Enable all rules by default, then selectively disable or configure
|
||||||
|
[rule.blank-imports]
|
||||||
|
[rule.context-as-argument]
|
||||||
|
[rule.context-keys-type]
|
||||||
|
[rule.dot-imports]
|
||||||
|
[rule.error-return]
|
||||||
|
[rule.error-strings]
|
||||||
|
[rule.error-naming]
|
||||||
|
[rule.exported]
|
||||||
|
[rule.if-return]
|
||||||
|
[rule.increment-decrement]
|
||||||
|
[rule.var-naming]
|
||||||
|
[rule.var-declaration]
|
||||||
|
[rule.package-comments]
|
||||||
|
[rule.range]
|
||||||
|
[rule.receiver-naming]
|
||||||
|
[rule.time-naming]
|
||||||
|
[rule.unexported-return]
|
||||||
|
[rule.indent-error-flow]
|
||||||
|
[rule.errorf]
|
||||||
|
[rule.empty-block]
|
||||||
|
[rule.superfluous-else]
|
||||||
|
[rule.unused-parameter]
|
||||||
|
[rule.unreachable-code]
|
||||||
|
[rule.redefines-builtin-id]
|
||||||
|
|
||||||
|
# Configure specific rules
|
||||||
|
[rule.line-length-limit]
|
||||||
|
arguments = [120]
|
||||||
|
Exclude = ["**/*_test.go"]
|
||||||
|
|
||||||
|
[rule.function-length]
|
||||||
|
arguments = [50, 100]
|
||||||
|
Exclude = ["**/*_test.go"]
|
||||||
|
|
||||||
|
[rule.max-public-structs]
|
||||||
|
arguments = [10]
|
||||||
|
|
||||||
|
[rule.cognitive-complexity]
|
||||||
|
arguments = [15]
|
||||||
|
Exclude = ["**/*_test.go"]
|
||||||
|
|
||||||
|
[rule.cyclomatic]
|
||||||
|
arguments = [15]
|
||||||
|
Exclude = ["**/*_test.go"]
|
||||||
|
|
||||||
|
[rule.argument-limit]
|
||||||
|
arguments = [5]
|
||||||
@@ -1,25 +1,31 @@
|
|||||||
Available targets:
|
Available targets:
|
||||||
install-tools - Install required linting and development tools
|
install-tools - Install required linting and development tools
|
||||||
lint - Run all linters (Go, Makefile, shell, YAML)
|
lint - Run all linters (Go, EditorConfig, Makefile, shell, YAML)
|
||||||
lint-fix - Run linters with auto-fix enabled
|
lint-fix - Run linters with auto-fix enabled
|
||||||
lint-verbose - Run linters with verbose output
|
lint-verbose - Run linters with verbose output
|
||||||
test - Run tests
|
test - Run tests
|
||||||
coverage - Run tests with coverage
|
test-coverage - Run tests with coverage output
|
||||||
build - Build the application
|
coverage - Run tests with coverage and generate HTML report
|
||||||
clean - Clean build artifacts
|
build - Build the application
|
||||||
all - Run lint, test, and build
|
clean - Clean build artifacts
|
||||||
|
all - Run lint, test, and build
|
||||||
|
|
||||||
Security targets:
|
Security targets:
|
||||||
security - Run comprehensive security scan
|
security - Run comprehensive security scan
|
||||||
security-full - Run full security analysis with all tools
|
security-full - Run full security analysis with all tools
|
||||||
vuln-check - Check for dependency vulnerabilities
|
vuln-check - Check for dependency vulnerabilities
|
||||||
|
|
||||||
|
Dependency management:
|
||||||
|
deps-check - Check for available dependency updates
|
||||||
|
deps-update - Update all dependencies to latest versions
|
||||||
|
deps-tidy - Clean up and verify dependencies
|
||||||
|
|
||||||
Benchmark targets:
|
Benchmark targets:
|
||||||
build-benchmark - Build the benchmark binary
|
build-benchmark - Build the benchmark binary
|
||||||
benchmark - Run all benchmarks
|
benchmark - Run all benchmarks
|
||||||
benchmark-collection - Run file collection benchmarks
|
benchmark-collection - Run file collection benchmarks
|
||||||
benchmark-processing - Run file processing benchmarks
|
benchmark-processing - Run file processing benchmarks
|
||||||
benchmark-concurrency - Run concurrency benchmarks
|
benchmark-concurrency - Run concurrency benchmarks
|
||||||
benchmark-format - Run format benchmarks
|
benchmark-format - Run format benchmarks
|
||||||
|
|
||||||
Run 'make <target>' to execute a specific target.
|
Run 'make <target>' to execute a specific target.
|
||||||
|
|||||||
@@ -1,14 +1,23 @@
|
|||||||
#!/bin/bash
|
#!/bin/sh
|
||||||
set -e
|
set -eu
|
||||||
|
|
||||||
echo "Running golangci-lint..."
|
echo "Running golangci-lint..."
|
||||||
golangci-lint run ./...
|
golangci-lint run ./...
|
||||||
|
|
||||||
|
echo "Running revive..."
|
||||||
|
revive -config revive.toml -formatter friendly ./...
|
||||||
|
|
||||||
echo "Running checkmake..."
|
echo "Running checkmake..."
|
||||||
checkmake --config=.checkmake Makefile
|
checkmake --config=.checkmake Makefile
|
||||||
|
|
||||||
|
echo "Running editorconfig-checker..."
|
||||||
|
editorconfig-checker
|
||||||
|
|
||||||
|
echo "Running shellcheck..."
|
||||||
|
shellcheck scripts/*.sh
|
||||||
|
|
||||||
echo "Running shfmt check..."
|
echo "Running shfmt check..."
|
||||||
shfmt -d .
|
shfmt -d -i 0 -ci .
|
||||||
|
|
||||||
echo "Running yamllint..."
|
echo "Running yamllint..."
|
||||||
yamllint -c .yamllint .
|
yamllint .
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
#!/bin/bash
|
#!/bin/sh
|
||||||
set -euo pipefail
|
set -eu
|
||||||
|
|
||||||
# Security Scanning Script for gibidify
|
# Security Scanning Script for gibidify
|
||||||
# This script runs comprehensive security checks locally and in CI
|
# This script runs comprehensive security checks locally and in CI
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
cd "$PROJECT_ROOT"
|
cd "$PROJECT_ROOT"
|
||||||
@@ -20,63 +20,177 @@ NC='\033[0m' # No Color
|
|||||||
|
|
||||||
# Function to print status
|
# Function to print status
|
||||||
print_status() {
|
print_status() {
|
||||||
echo -e "${BLUE}[INFO]${NC} $1"
|
printf "${BLUE}[INFO]${NC} %s\n" "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
print_warning() {
|
print_warning() {
|
||||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
printf "${YELLOW}[WARN]${NC} %s\n" "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
print_error() {
|
print_error() {
|
||||||
echo -e "${RED}[ERROR]${NC} $1"
|
printf "${RED}[ERROR]${NC} %s\n" "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
print_success() {
|
print_success() {
|
||||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
printf "${GREEN}[SUCCESS]${NC} %s\n" "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run command with timeout if available, otherwise run directly
|
||||||
|
# Usage: run_with_timeout DURATION COMMAND [ARGS...]
|
||||||
|
run_with_timeout() {
|
||||||
|
duration="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
if command -v timeout >/dev/null 2>&1; then
|
||||||
|
timeout "$duration" "$@"
|
||||||
|
else
|
||||||
|
# timeout not available, run command directly
|
||||||
|
"$@"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Check if required tools are installed
|
# Check if required tools are installed
|
||||||
check_dependencies() {
|
check_dependencies() {
|
||||||
print_status "Checking security scanning dependencies..."
|
print_status "Checking security scanning dependencies..."
|
||||||
|
|
||||||
local missing_tools=()
|
missing_tools=""
|
||||||
|
|
||||||
if ! command -v go &>/dev/null; then
|
if ! command -v go >/dev/null 2>&1; then
|
||||||
missing_tools+=("go")
|
missing_tools="${missing_tools}go "
|
||||||
|
print_error "Go is not installed. Please install Go first."
|
||||||
|
print_error "Visit https://golang.org/doc/install for installation instructions."
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v golangci-lint &>/dev/null; then
|
if ! command -v golangci-lint >/dev/null 2>&1; then
|
||||||
print_warning "golangci-lint not found, installing..."
|
print_warning "golangci-lint not found, installing..."
|
||||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v gosec &>/dev/null; then
|
if ! command -v gosec >/dev/null 2>&1; then
|
||||||
print_warning "gosec not found, installing..."
|
print_warning "gosec not found, installing..."
|
||||||
go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest
|
go install github.com/securego/gosec/v2/cmd/gosec@latest
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v govulncheck &>/dev/null; then
|
if ! command -v govulncheck >/dev/null 2>&1; then
|
||||||
print_warning "govulncheck not found, installing..."
|
print_warning "govulncheck not found, installing..."
|
||||||
go install golang.org/x/vuln/cmd/govulncheck@latest
|
go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v checkmake &>/dev/null; then
|
if ! command -v checkmake >/dev/null 2>&1; then
|
||||||
print_warning "checkmake not found, installing..."
|
print_warning "checkmake not found, installing..."
|
||||||
go install github.com/mrtazz/checkmake/cmd/checkmake@latest
|
go install github.com/checkmake/checkmake/cmd/checkmake@latest
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v shfmt &>/dev/null; then
|
if ! command -v shfmt >/dev/null 2>&1; then
|
||||||
print_warning "shfmt not found, installing..."
|
print_warning "shfmt not found, installing..."
|
||||||
go install mvdan.cc/sh/v3/cmd/shfmt@latest
|
go install mvdan.cc/sh/v3/cmd/shfmt@latest
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v yamllint &>/dev/null; then
|
if ! command -v yamllint >/dev/null 2>&1; then
|
||||||
print_warning "yamllint not found, installing..."
|
print_warning "yamllint not found, attempting to install..."
|
||||||
go install github.com/excilsploft/yamllint@latest
|
|
||||||
|
# Update PATH to include common user install directories
|
||||||
|
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||||
|
|
||||||
|
installed=0
|
||||||
|
|
||||||
|
# Try pipx first
|
||||||
|
if command -v pipx >/dev/null 2>&1; then
|
||||||
|
print_status "Attempting install with pipx..."
|
||||||
|
if pipx install yamllint; then
|
||||||
|
# Update PATH to include pipx bin directory
|
||||||
|
pipx_bin_dir=$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")
|
||||||
|
export PATH="$pipx_bin_dir:$PATH"
|
||||||
|
installed=1
|
||||||
|
else
|
||||||
|
print_warning "pipx install yamllint failed, trying next method..."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Try pip3 --user if pipx didn't work
|
||||||
|
if [ "$installed" -eq 0 ] && command -v pip3 >/dev/null 2>&1; then
|
||||||
|
print_status "Attempting install with pip3 --user..."
|
||||||
|
if pip3 install --user yamllint; then
|
||||||
|
installed=1
|
||||||
|
else
|
||||||
|
print_warning "pip3 install yamllint failed, trying next method..."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Try apt-get with smart sudo handling
|
||||||
|
if [ "$installed" -eq 0 ] && command -v apt-get >/dev/null 2>&1; then
|
||||||
|
sudo_cmd=""
|
||||||
|
can_use_apt=false
|
||||||
|
|
||||||
|
# Check if running as root
|
||||||
|
if [ "$(id -u)" -eq 0 ]; then
|
||||||
|
print_status "Running as root, no sudo needed for apt-get..."
|
||||||
|
sudo_cmd=""
|
||||||
|
can_use_apt=true
|
||||||
|
elif command -v sudo >/dev/null 2>&1; then
|
||||||
|
# Try non-interactive sudo first
|
||||||
|
if sudo -n true 2>/dev/null; then
|
||||||
|
print_status "Attempting install with apt-get (sudo cached)..."
|
||||||
|
sudo_cmd="sudo"
|
||||||
|
can_use_apt=true
|
||||||
|
elif [ -t 0 ]; then
|
||||||
|
# TTY available, allow interactive sudo
|
||||||
|
print_status "Attempting install with apt-get (may prompt for sudo)..."
|
||||||
|
sudo_cmd="sudo"
|
||||||
|
can_use_apt=true
|
||||||
|
else
|
||||||
|
print_warning "apt-get available but sudo not accessible (non-interactive, no cache). Skipping apt-get."
|
||||||
|
can_use_apt=false
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
print_warning "apt-get available but sudo not found. Skipping apt-get."
|
||||||
|
can_use_apt=false
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Attempt apt-get only if we have permission to use it
|
||||||
|
if [ "$can_use_apt" = true ]; then
|
||||||
|
if [ -n "$sudo_cmd" ]; then
|
||||||
|
if run_with_timeout 300 ${sudo_cmd:+"$sudo_cmd"} apt-get update; then
|
||||||
|
if run_with_timeout 300 ${sudo_cmd:+"$sudo_cmd"} apt-get install -y yamllint; then
|
||||||
|
installed=1
|
||||||
|
else
|
||||||
|
print_warning "apt-get install yamllint failed or timed out"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
print_warning "apt-get update failed or timed out"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Running as root without sudo
|
||||||
|
if run_with_timeout 300 apt-get update; then
|
||||||
|
if run_with_timeout 300 apt-get install -y yamllint; then
|
||||||
|
installed=1
|
||||||
|
else
|
||||||
|
print_warning "apt-get install yamllint failed or timed out"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
print_warning "apt-get update failed or timed out"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Final check with updated PATH
|
||||||
|
if ! command -v yamllint >/dev/null 2>&1; then
|
||||||
|
print_error "yamllint installation failed or yamllint still not found in PATH."
|
||||||
|
print_error "Please install yamllint manually using one of:"
|
||||||
|
print_error " - pipx install yamllint"
|
||||||
|
print_error " - pip3 install --user yamllint"
|
||||||
|
print_error " - sudo apt-get install yamllint (Debian/Ubuntu)"
|
||||||
|
print_error " - brew install yamllint (macOS)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
print_status "yamllint successfully installed and found in PATH"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ${#missing_tools[@]} -ne 0 ]; then
|
if [ -n "$missing_tools" ]; then
|
||||||
print_error "Missing required tools: ${missing_tools[*]}"
|
print_error "Missing required tools: $missing_tools"
|
||||||
print_error "Please install the missing tools and try again."
|
print_error "Please install the missing tools and try again."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -103,15 +217,41 @@ run_gosec() {
|
|||||||
run_govulncheck() {
|
run_govulncheck() {
|
||||||
print_status "Running govulncheck for dependency vulnerabilities..."
|
print_status "Running govulncheck for dependency vulnerabilities..."
|
||||||
|
|
||||||
if govulncheck -json ./... >govulncheck-report.json 2>&1; then
|
# govulncheck with -json always exits 0, so we need to check the output
|
||||||
print_success "No known vulnerabilities found in dependencies"
|
# Redirect stderr to separate file to avoid corrupting JSON output
|
||||||
else
|
govulncheck -json ./... >govulncheck-report.json 2>govulncheck-errors.log
|
||||||
if grep -q '"finding"' govulncheck-report.json 2>/dev/null; then
|
|
||||||
|
# Check if there were errors during execution
|
||||||
|
if [ -s govulncheck-errors.log ]; then
|
||||||
|
print_warning "govulncheck produced errors (see govulncheck-errors.log)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use jq to detect finding entries in the JSON output
|
||||||
|
# govulncheck emits a stream of Message objects, need to slurp and filter for Finding field
|
||||||
|
if command -v jq >/dev/null 2>&1; then
|
||||||
|
# First validate JSON is parseable
|
||||||
|
if ! jq -s '.' govulncheck-report.json >/dev/null 2>&1; then
|
||||||
|
print_error "govulncheck report contains malformed JSON"
|
||||||
|
echo "Unable to parse govulncheck-report.json"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# JSON is valid, now check for findings
|
||||||
|
if jq -s -e 'map(select(.Finding)) | length > 0' govulncheck-report.json >/dev/null 2>&1; then
|
||||||
print_error "Vulnerabilities found in dependencies!"
|
print_error "Vulnerabilities found in dependencies!"
|
||||||
echo "Detailed report saved to govulncheck-report.json"
|
echo "Detailed report saved to govulncheck-report.json"
|
||||||
return 1
|
return 1
|
||||||
else
|
else
|
||||||
print_success "No vulnerabilities found"
|
print_success "No known vulnerabilities found in dependencies"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Fallback to grep if jq is not available (case-insensitive to match "Finding")
|
||||||
|
if grep -qi '"finding":' govulncheck-report.json 2>/dev/null; then
|
||||||
|
print_error "Vulnerabilities found in dependencies!"
|
||||||
|
echo "Detailed report saved to govulncheck-report.json"
|
||||||
|
return 1
|
||||||
|
else
|
||||||
|
print_success "No known vulnerabilities found in dependencies"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
@@ -120,7 +260,7 @@ run_govulncheck() {
|
|||||||
run_security_lint() {
|
run_security_lint() {
|
||||||
print_status "Running security-focused linting..."
|
print_status "Running security-focused linting..."
|
||||||
|
|
||||||
local security_linters="gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused,errcheck,ineffassign,staticcheck"
|
security_linters="gosec,gocritic,bodyclose,rowserrcheck,misspell,unconvert,unparam,unused,errcheck,ineffassign,staticcheck"
|
||||||
|
|
||||||
if golangci-lint run --enable="$security_linters" --timeout=5m; then
|
if golangci-lint run --enable="$security_linters" --timeout=5m; then
|
||||||
print_success "Security linting passed"
|
print_success "Security linting passed"
|
||||||
@@ -134,31 +274,47 @@ run_security_lint() {
|
|||||||
check_secrets() {
|
check_secrets() {
|
||||||
print_status "Scanning for potential secrets and sensitive data..."
|
print_status "Scanning for potential secrets and sensitive data..."
|
||||||
|
|
||||||
local secrets_found=false
|
# POSIX-compatible secrets_found flag using a temp file
|
||||||
|
secrets_found_file="$(mktemp)" || {
|
||||||
|
print_error "Failed to create temporary file with mktemp"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
if [ -z "$secrets_found_file" ]; then
|
||||||
|
print_error "mktemp returned empty path"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Clean up temp file on exit and signals (POSIX-portable)
|
||||||
|
trap 'rm -f "$secrets_found_file"' 0 HUP INT TERM
|
||||||
|
|
||||||
# Common secret patterns
|
# Common secret patterns (POSIX [[:space:]] and here-doc quoting)
|
||||||
local patterns=(
|
cat <<'PATTERNS' | while IFS= read -r pattern; do
|
||||||
"password\s*[:=]\s*['\"][^'\"]{3,}['\"]"
|
password[[:space:]]*[:=][[:space:]]*['"][^'"]{3,}['"]
|
||||||
"secret\s*[:=]\s*['\"][^'\"]{3,}['\"]"
|
secret[[:space:]]*[:=][[:space:]]*['"][^'"]{3,}['"]
|
||||||
"key\s*[:=]\s*['\"][^'\"]{8,}['\"]"
|
key[[:space:]]*[:=][[:space:]]*['"][^'"]{8,}['"]
|
||||||
"token\s*[:=]\s*['\"][^'\"]{8,}['\"]"
|
token[[:space:]]*[:=][[:space:]]*['"][^'"]{8,}['"]
|
||||||
"api_?key\s*[:=]\s*['\"][^'\"]{8,}['\"]"
|
api_?key[[:space:]]*[:=][[:space:]]*['"][^'"]{8,}['"]
|
||||||
"aws_?access_?key"
|
aws_?access_?key
|
||||||
"aws_?secret"
|
aws_?secret
|
||||||
"AKIA[0-9A-Z]{16}" # AWS Access Key pattern
|
AKIA[0-9A-Z]{16}
|
||||||
"github_?token"
|
github_?token
|
||||||
"private_?key"
|
private_?key
|
||||||
)
|
PATTERNS
|
||||||
|
if [ -n "$pattern" ]; then
|
||||||
for pattern in "${patterns[@]}"; do
|
if find . -type f -name "*.go" -exec grep -i -E -H -n -e "$pattern" {} + 2>/dev/null | grep -q .; then
|
||||||
if grep -r -i -E "$pattern" --include="*.go" . 2>/dev/null; then
|
print_warning "Potential secret pattern found: $pattern"
|
||||||
print_warning "Potential secret pattern found: $pattern"
|
touch "$secrets_found_file"
|
||||||
secrets_found=true
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if [ -f "$secrets_found_file" ]; then
|
||||||
|
secrets_found=true
|
||||||
|
else
|
||||||
|
secrets_found=false
|
||||||
|
fi
|
||||||
|
|
||||||
# Check git history for secrets (last 10 commits)
|
# Check git history for secrets (last 10 commits)
|
||||||
if git log --oneline -10 | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then
|
if git log --oneline -10 2>/dev/null | grep -i -E "(password|secret|key|token)" >/dev/null 2>&1; then
|
||||||
print_warning "Potential secrets mentioned in recent commit messages"
|
print_warning "Potential secrets mentioned in recent commit messages"
|
||||||
secrets_found=true
|
secrets_found=true
|
||||||
fi
|
fi
|
||||||
@@ -175,23 +331,23 @@ check_secrets() {
|
|||||||
check_hardcoded_addresses() {
|
check_hardcoded_addresses() {
|
||||||
print_status "Checking for hardcoded network addresses..."
|
print_status "Checking for hardcoded network addresses..."
|
||||||
|
|
||||||
local addresses_found=false
|
addresses_found=false
|
||||||
|
|
||||||
# Look for IP addresses (excluding common safe ones)
|
# Look for IP addresses (excluding common safe ones)
|
||||||
if grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . |
|
if grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . 2>/dev/null |
|
||||||
grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" >/dev/null 2>&1; then
|
grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" >/dev/null 2>&1; then
|
||||||
print_warning "Hardcoded IP addresses found:"
|
print_warning "Hardcoded IP addresses found:"
|
||||||
grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . |
|
grep -r -E "([0-9]{1,3}\.){3}[0-9]{1,3}" --include="*.go" . 2>/dev/null |
|
||||||
grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" || true
|
grep -v -E "(127\.0\.0\.1|0\.0\.0\.0|255\.255\.255\.255|localhost)" || true
|
||||||
addresses_found=true
|
addresses_found=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Look for URLs (excluding documentation examples)
|
# Look for URLs (excluding documentation examples and comments)
|
||||||
if grep -r -E "https?://[^/\s]+" --include="*.go" . |
|
if grep -r -E "https?://[^/\s]+" --include="*.go" . 2>/dev/null |
|
||||||
grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{)" >/dev/null 2>&1; then
|
grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{|//.*https?://)" >/dev/null 2>&1; then
|
||||||
print_warning "Hardcoded URLs found:"
|
print_warning "Hardcoded URLs found:"
|
||||||
grep -r -E "https?://[^/\s]+" --include="*.go" . |
|
grep -r -E "https?://[^/\s]+" --include="*.go" . 2>/dev/null |
|
||||||
grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{)" || true
|
grep -v -E "(example\.com|localhost|127\.0\.0\.1|\$\{|//.*https?://)" || true
|
||||||
addresses_found=true
|
addresses_found=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -209,7 +365,7 @@ check_docker_security() {
|
|||||||
print_status "Checking Docker security..."
|
print_status "Checking Docker security..."
|
||||||
|
|
||||||
# Basic Dockerfile security checks
|
# Basic Dockerfile security checks
|
||||||
local docker_issues=false
|
docker_issues=false
|
||||||
|
|
||||||
if grep -q "^USER root" Dockerfile; then
|
if grep -q "^USER root" Dockerfile; then
|
||||||
print_warning "Dockerfile runs as root user"
|
print_warning "Dockerfile runs as root user"
|
||||||
@@ -221,7 +377,7 @@ check_docker_security() {
|
|||||||
docker_issues=true
|
docker_issues=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if grep -q "RUN.*wget\|RUN.*curl" Dockerfile && ! grep -q "rm.*wget\|rm.*curl" Dockerfile; then
|
if grep -Eq 'RUN.*(wget|curl)' Dockerfile && ! grep -Eq 'rm.*(wget|curl)' Dockerfile; then
|
||||||
print_warning "Dockerfile may leave curl/wget installed"
|
print_warning "Dockerfile may leave curl/wget installed"
|
||||||
docker_issues=true
|
docker_issues=true
|
||||||
fi
|
fi
|
||||||
@@ -241,19 +397,21 @@ check_docker_security() {
|
|||||||
check_file_permissions() {
|
check_file_permissions() {
|
||||||
print_status "Checking file permissions..."
|
print_status "Checking file permissions..."
|
||||||
|
|
||||||
local perm_issues=false
|
perm_issues=false
|
||||||
|
|
||||||
# Check for overly permissive files
|
# Check for overly permissive files (using octal for cross-platform compatibility)
|
||||||
if find . -type f -perm /o+w -not -path "./.git/*" | grep -q .; then
|
# -perm -002 finds files writable by others (works on both BSD and GNU find)
|
||||||
|
if find . -type f -perm -002 -not -path "./.git/*" 2>/dev/null | grep -q .; then
|
||||||
print_warning "World-writable files found:"
|
print_warning "World-writable files found:"
|
||||||
find . -type f -perm /o+w -not -path "./.git/*" || true
|
find . -type f -perm -002 -not -path "./.git/*" 2>/dev/null || true
|
||||||
perm_issues=true
|
perm_issues=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check for executable files that shouldn't be
|
# Check for executable files that shouldn't be
|
||||||
if find . -type f -name "*.go" -perm /a+x | grep -q .; then
|
# -perm -111 finds files executable by anyone (works on both BSD and GNU find)
|
||||||
|
if find . -type f -name "*.go" -perm -111 -not -path "./.git/*" 2>/dev/null | grep -q .; then
|
||||||
print_warning "Executable Go files found (should not be executable):"
|
print_warning "Executable Go files found (should not be executable):"
|
||||||
find . -type f -name "*.go" -perm /a+x || true
|
find . -type f -name "*.go" -perm -111 -not -path "./.git/*" 2>/dev/null || true
|
||||||
perm_issues=true
|
perm_issues=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -285,7 +443,7 @@ check_makefile() {
|
|||||||
check_shell_scripts() {
|
check_shell_scripts() {
|
||||||
print_status "Checking shell script formatting..."
|
print_status "Checking shell script formatting..."
|
||||||
|
|
||||||
if find . -name "*.sh" -type f | head -1 | grep -q .; then
|
if find . -name "*.sh" -type f 2>/dev/null | head -1 | grep -q .; then
|
||||||
if shfmt -d .; then
|
if shfmt -d .; then
|
||||||
print_success "Shell script formatting check passed"
|
print_success "Shell script formatting check passed"
|
||||||
else
|
else
|
||||||
@@ -301,8 +459,8 @@ check_shell_scripts() {
|
|||||||
check_yaml_files() {
|
check_yaml_files() {
|
||||||
print_status "Checking YAML files..."
|
print_status "Checking YAML files..."
|
||||||
|
|
||||||
if find . -name "*.yml" -o -name "*.yaml" -type f | head -1 | grep -q .; then
|
if find . \( -name "*.yml" -o -name "*.yaml" \) -type f 2>/dev/null | head -1 | grep -q .; then
|
||||||
if yamllint -c .yamllint .; then
|
if yamllint .; then
|
||||||
print_success "YAML files check passed"
|
print_success "YAML files check passed"
|
||||||
else
|
else
|
||||||
print_error "YAML file issues detected!"
|
print_error "YAML file issues detected!"
|
||||||
@@ -317,7 +475,7 @@ check_yaml_files() {
|
|||||||
generate_report() {
|
generate_report() {
|
||||||
print_status "Generating security scan report..."
|
print_status "Generating security scan report..."
|
||||||
|
|
||||||
local report_file="security-report.md"
|
report_file="security-report.md"
|
||||||
|
|
||||||
cat >"$report_file" <<EOF
|
cat >"$report_file" <<EOF
|
||||||
# Security Scan Report
|
# Security Scan Report
|
||||||
@@ -370,7 +528,7 @@ main() {
|
|||||||
echo "=========================="
|
echo "=========================="
|
||||||
echo
|
echo
|
||||||
|
|
||||||
local exit_code=0
|
exit_code=0
|
||||||
|
|
||||||
check_dependencies
|
check_dependencies
|
||||||
echo
|
echo
|
||||||
@@ -409,7 +567,7 @@ main() {
|
|||||||
generate_report
|
generate_report
|
||||||
echo
|
echo
|
||||||
|
|
||||||
if [ $exit_code -eq 0 ]; then
|
if [ "$exit_code" -eq 0 ]; then
|
||||||
print_success "🎉 All security checks passed!"
|
print_success "🎉 All security checks passed!"
|
||||||
else
|
else
|
||||||
print_error "❌ Security issues detected. Please review the reports and fix identified issues."
|
print_error "❌ Security issues detected. Please review the reports and fix identified issues."
|
||||||
@@ -419,7 +577,7 @@ main() {
|
|||||||
print_status "- security-report.md"
|
print_status "- security-report.md"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exit $exit_code
|
exit "$exit_code"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Run main function
|
# Run main function
|
||||||
|
|||||||
@@ -83,4 +83,4 @@ func BenchmarkVerifyContentContains(b *testing.B) {
|
|||||||
_ = strings.Contains(content, exp)
|
_ = strings.Contains(content, exp)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -129,4 +129,4 @@ func TestSetupCLIArgs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ func TestCreateTestFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify content
|
// Verify content
|
||||||
readContent, err := os.ReadFile(filePath)
|
readContent, err := os.ReadFile(filePath) // #nosec G304 - test file path is controlled
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to read created file: %v", err)
|
t.Fatalf("Failed to read created file: %v", err)
|
||||||
}
|
}
|
||||||
@@ -272,7 +272,7 @@ func TestCreateTestFiles(t *testing.T) {
|
|||||||
|
|
||||||
// Verify each file
|
// Verify each file
|
||||||
for i, filePath := range createdFiles {
|
for i, filePath := range createdFiles {
|
||||||
content, err := os.ReadFile(filePath)
|
content, err := os.ReadFile(filePath) // #nosec G304 - test file path is controlled
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Failed to read file %s: %v", filePath, err)
|
t.Errorf("Failed to read file %s: %v", filePath, err)
|
||||||
continue
|
continue
|
||||||
@@ -283,4 +283,4 @@ func TestCreateTestFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,10 +31,10 @@ func TestVerifyContentContains(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// This would normally fail but we're just checking it doesn't panic
|
// This would normally fail, but we're just checking it doesn't panic
|
||||||
content := "test"
|
content := "test"
|
||||||
expected := []string{"not found"}
|
expected := []string{"not found"}
|
||||||
// Create a sub-test that we expect to fail
|
// Create a subtest that we expect to fail
|
||||||
t.Run("expected_failure", func(t *testing.T) {
|
t.Run("expected_failure", func(t *testing.T) {
|
||||||
t.Skip("Skipping actual failure test")
|
t.Skip("Skipping actual failure test")
|
||||||
VerifyContentContains(t, content, expected)
|
VerifyContentContains(t, content, expected)
|
||||||
@@ -59,7 +59,7 @@ func TestMustSucceed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Create a sub-test that we expect to fail
|
// Create a subtest that we expect to fail
|
||||||
t.Run("expected_failure", func(t *testing.T) {
|
t.Run("expected_failure", func(t *testing.T) {
|
||||||
t.Skip("Skipping actual failure test")
|
t.Skip("Skipping actual failure test")
|
||||||
MustSucceed(t, errors.New("test error"), "failed operation")
|
MustSucceed(t, errors.New("test error"), "failed operation")
|
||||||
@@ -104,4 +104,4 @@ func TestCloseFile(t *testing.T) {
|
|||||||
t.Error("Expected write to fail on closed file")
|
t.Error("Expected write to fail on closed file")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
167
utils/paths.go
167
utils/paths.go
@@ -1,167 +0,0 @@
|
|||||||
// Package utils provides common utility functions.
|
|
||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GetAbsolutePath returns the absolute path for the given path.
|
|
||||||
// It wraps filepath.Abs with consistent error handling.
|
|
||||||
func GetAbsolutePath(path string) (string, error) {
|
|
||||||
abs, err := filepath.Abs(path)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to get absolute path for %s: %w", path, err)
|
|
||||||
}
|
|
||||||
return abs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetBaseName returns the base name for the given path, handling special cases.
|
|
||||||
func GetBaseName(absPath string) string {
|
|
||||||
baseName := filepath.Base(absPath)
|
|
||||||
if baseName == "." || baseName == "" {
|
|
||||||
return "output"
|
|
||||||
}
|
|
||||||
return baseName
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateSourcePath validates a source directory path for security.
|
|
||||||
// It ensures the path exists, is a directory, and doesn't contain path traversal attempts.
|
|
||||||
func ValidateSourcePath(path string) error {
|
|
||||||
if path == "" {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationRequired, "source path is required", "", nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for path traversal patterns before cleaning
|
|
||||||
if strings.Contains(path, "..") {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "path traversal attempt detected in source path", path, map[string]interface{}{
|
|
||||||
"original_path": path,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean and get absolute path
|
|
||||||
cleaned := filepath.Clean(path)
|
|
||||||
abs, err := filepath.Abs(cleaned)
|
|
||||||
if err != nil {
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot resolve source path", path, map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get current working directory to ensure we're not escaping it for relative paths
|
|
||||||
if !filepath.IsAbs(path) {
|
|
||||||
cwd, err := os.Getwd()
|
|
||||||
if err != nil {
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot get current working directory", path, map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the resolved path is within or below the current working directory
|
|
||||||
cwdAbs, err := filepath.Abs(cwd)
|
|
||||||
if err != nil {
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot resolve current working directory", path, map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the absolute path tries to escape the current working directory
|
|
||||||
if !strings.HasPrefix(abs, cwdAbs) {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "source path attempts to access directories outside current working directory", path, map[string]interface{}{
|
|
||||||
"resolved_path": abs,
|
|
||||||
"working_dir": cwdAbs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if path exists and is a directory
|
|
||||||
info, err := os.Stat(cleaned)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSNotFound, "source directory does not exist", path, nil)
|
|
||||||
}
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSAccess, "cannot access source directory", path, map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if !info.IsDir() {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "source path must be a directory", path, map[string]interface{}{
|
|
||||||
"is_file": true,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateDestinationPath validates a destination file path for security.
|
|
||||||
// It ensures the path doesn't contain path traversal attempts and the parent directory exists.
|
|
||||||
func ValidateDestinationPath(path string) error {
|
|
||||||
if path == "" {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationRequired, "destination path is required", "", nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for path traversal patterns before cleaning
|
|
||||||
if strings.Contains(path, "..") {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "path traversal attempt detected in destination path", path, map[string]interface{}{
|
|
||||||
"original_path": path,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean and validate the path
|
|
||||||
cleaned := filepath.Clean(path)
|
|
||||||
|
|
||||||
// Get absolute path to ensure it's not trying to escape current working directory
|
|
||||||
abs, err := filepath.Abs(cleaned)
|
|
||||||
if err != nil {
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSPathResolution, "cannot resolve destination path", path, map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the destination is not a directory
|
|
||||||
if info, err := os.Stat(abs); err == nil && info.IsDir() {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "destination cannot be a directory", path, map[string]interface{}{
|
|
||||||
"is_directory": true,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if parent directory exists and is writable
|
|
||||||
parentDir := filepath.Dir(abs)
|
|
||||||
if parentInfo, err := os.Stat(parentDir); err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSNotFound, "destination parent directory does not exist", path, map[string]interface{}{
|
|
||||||
"parent_dir": parentDir,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return NewStructuredError(ErrorTypeFileSystem, CodeFSAccess, "cannot access destination parent directory", path, map[string]interface{}{
|
|
||||||
"parent_dir": parentDir,
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
} else if !parentInfo.IsDir() {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "destination parent is not a directory", path, map[string]interface{}{
|
|
||||||
"parent_dir": parentDir,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateConfigPath validates a configuration file path for security.
|
|
||||||
// It ensures the path doesn't contain path traversal attempts.
|
|
||||||
func ValidateConfigPath(path string) error {
|
|
||||||
if path == "" {
|
|
||||||
return nil // Empty path is allowed for config
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for path traversal patterns before cleaning
|
|
||||||
if strings.Contains(path, "..") {
|
|
||||||
return NewStructuredError(ErrorTypeValidation, CodeValidationPath, "path traversal attempt detected in config path", path, map[string]interface{}{
|
|
||||||
"original_path": path,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user