diff --git a/.github/workflows/remind-sync-docs.yml b/.github/workflows/remind-sync-docs.yml new file mode 100644 index 0000000..85a437d --- /dev/null +++ b/.github/workflows/remind-sync-docs.yml @@ -0,0 +1,122 @@ +name: Remind to Sync Documentation + +on: + push: + branches: [main] + paths: + - 'influxdata/**/README.md' + +permissions: + contents: read + +jobs: + remind-sync: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 # Need previous commit to detect changes + + - name: Detect changed plugins + id: detect-changes + run: | + # Get list of changed README files + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep '^influxdata/.*/README\.md$') + + if [[ -z "$CHANGED_FILES" ]]; then + echo "No plugin README files changed" + echo "changed_plugins=" >> $GITHUB_OUTPUT + echo "has_changes=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Changed files:" + echo "$CHANGED_FILES" + + # Extract plugin names from file paths + PLUGIN_NAMES="" + while IFS= read -r file; do + if [[ -n "$file" ]]; then + # Extract plugin name from path: influxdata/plugin_name/README.md -> plugin_name + PLUGIN_NAME=$(echo "$file" | sed 's|influxdata/||' | sed 's|/README\.md||') + if [[ -n "$PLUGIN_NAMES" ]]; then + PLUGIN_NAMES="$PLUGIN_NAMES, $PLUGIN_NAME" + else + PLUGIN_NAMES="$PLUGIN_NAME" + fi + fi + done <<< "$CHANGED_FILES" + + echo "Changed plugins: $PLUGIN_NAMES" + echo "changed_plugins=$PLUGIN_NAMES" >> $GITHUB_OUTPUT + echo "has_changes=true" >> $GITHUB_OUTPUT + + - name: Create sync reminder comment + if: steps.detect-changes.outputs.has_changes == 'true' + uses: actions/github-script@v7 + with: + script: | + const changedPlugins = '${{ steps.detect-changes.outputs.changed_plugins }}'; + const commitSha = context.sha; + const shortSha = commitSha.substring(0, 7); + + // Build the GitHub issue URL with pre-filled parameters + const baseUrl = 'https://github.com/influxdata/docs-v2/issues/new'; + const template = 'sync-plugin-docs.yml'; + const title = encodeURIComponent(`Sync plugin docs: ${changedPlugins}`); + const plugins = encodeURIComponent(changedPlugins); + const sourceCommit = encodeURIComponent(commitSha); + + const issueUrl = `${baseUrl}?template=${template}&title=${title}&plugins=${plugins}&source_commit=${sourceCommit}`; + + // Create the comment body + const commentBody = `šŸ“š **Plugin documentation updated!** + + The following plugin READMEs were changed in this commit: + **${changedPlugins}** + + ## Next Steps + + To sync these changes to the InfluxDB documentation site: + + ### šŸš€ [**Click here to create sync request**](${issueUrl}) + + This will open a pre-filled issue in docs-v2 that will automatically trigger the sync workflow. + + ### What the sync process does: + 1. āœ… Validates your plugin READMEs against template requirements + 2. šŸ”„ Transforms content for docs-v2 compatibility (adds Hugo shortcodes, fixes links) + 3. šŸ–¼ļø Generates screenshots of the plugin documentation pages + 4. šŸ“ Creates a pull request in docs-v2 ready for review + + ### Before syncing: + - Ensure your README follows the [README_TEMPLATE.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README_TEMPLATE.md) structure + - Include proper emoji metadata (āš” triggers, šŸ·ļø tags, šŸ”§ compatibility) + - Verify all required sections are present and complete + + --- + *Commit: ${shortSha} | [View workflow](https://github.com/influxdata/docs-v2/blob/master/helper-scripts/influxdb3-plugins/README.md)*`; + + // Create commit comment + await github.rest.repos.createCommitComment({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: commitSha, + body: commentBody + }); + + console.log(`Created sync reminder for plugins: ${changedPlugins}`); + console.log(`Issue URL: ${issueUrl}`); + + - name: Log workflow completion + if: steps.detect-changes.outputs.has_changes == 'true' + run: | + echo "āœ… Sync reminder created for plugins: ${{ steps.detect-changes.outputs.changed_plugins }}" + echo "šŸ”— Users can click the link in the commit comment to trigger docs sync" + + - name: No changes detected + if: steps.detect-changes.outputs.has_changes == 'false' + run: | + echo "ā„¹ļø No plugin README files were changed in this commit" \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0d093a4..8b516bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -489,6 +489,51 @@ raise ValueError( ) ``` +## Documentation Sync Process + +### Syncing Plugin Documentation to docs-v2 + +When you update plugin READMEs in this repository, they need to be synchronized to the InfluxDB documentation site (docs-v2) to appear on docs.influxdata.com. + +#### Automated Sync Workflow + +1. **Make your changes** - Update plugin README files following the template structure +2. **Commit and push** - Push your changes to the master branch +3. **Check for reminder** - A workflow will automatically detect README changes and create a commit comment with a sync link +4. **Click sync link** - The comment includes a pre-filled link to create a sync request in docs-v2 +5. **Submit request** - This automatically triggers validation, transformation, and PR creation +6. **Review PR** - The resulting PR in docs-v2 includes screenshots and change summaries + +#### Manual Sync (Alternative) + +You can also manually trigger synchronization: + +1. **Navigate to sync form**: [Create sync request](https://github.com/influxdata/docs-v2/issues/new?template=sync-plugin-docs.yml) +2. **Fill in details** - Specify plugin names and source commit +3. **Submit** - The automation workflow handles the rest + +#### Sync Requirements + +Before syncing, ensure your README: + +- āœ… Follows the [README_TEMPLATE.md](README_TEMPLATE.md) structure +- āœ… Includes proper emoji metadata (`āš”` triggers, `šŸ·ļø` tags, `šŸ”§` compatibility) +- āœ… Has all required sections with proper formatting +- āœ… Contains working examples with expected output +- āœ… Passes validation (`python scripts/validate_readme.py`) + +#### What Gets Transformed + +During sync, your README content is automatically transformed for docs-v2: + +- **Emoji metadata removed** (already in plugin JSON metadata) +- **Relative links converted** to GitHub URLs +- **Product references enhanced** with Hugo shortcodes (`{{% product-name %}}`) +- **Logging section added** with standard content +- **Support sections updated** with docs-v2 format + +Your original README remains unchanged - these transformations only apply to the docs-v2 copy. + ## Commit Message Format ### Use conventional commits @@ -509,4 +554,17 @@ raise ValueError( - `test`: Test changes - `chore`: Maintenance tasks +### Documentation Sync Messages + +When making documentation-focused commits, use clear messages that describe what changed: + +```bash +# Good commit messages for docs sync +docs(basic_transformation): update configuration parameters and examples +feat(downsampler): add support for custom aggregation functions +fix(notifier): correct email configuration example + +# These will trigger sync reminders automatically +``` + *These standards are extracted from the [InfluxData Documentation guidelines](https://github.com/influxdata/docs-v2/blob/master/CONTRIBUTING.md).* diff --git a/README_TEMPLATE.md b/README_TEMPLATE.md new file mode 100644 index 0000000..de8777f --- /dev/null +++ b/README_TEMPLATE.md @@ -0,0 +1,350 @@ +# Plugin Name + +āš” scheduled, data-write, http šŸ·ļø tag1, tag2, tag3 šŸ”§ InfluxDB 3 Core, InfluxDB 3 Enterprise + +## Description + +Brief description of what the plugin does and its primary use case. Include the trigger types supported (write, scheduled, HTTP) and main functionality. Mention any special features or capabilities that distinguish this plugin. Add a fourth sentence if needed for additional context. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. Some plugins support TOML configuration files, which can be specified using the plugin's `config_file_path` parameter. + +If a plugin supports multiple trigger specifications, some parameters may depend on the trigger specification that you use. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `parameter_name` | string | required | Description of the parameter | +| `another_param` | integer | required | Description with any constraints or requirements | + +### Optional parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `optional_param` | boolean | "false" | Description of optional parameter | +| `timeout` | integer | 30 | Connection timeout in seconds | + +### [Category] parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `category_param` | string | "default" | Parameters grouped by functionality | + +### TOML configuration + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | + +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. + +#### Example TOML configurations + +- [plugin_config_scheduler.toml](plugin_config_scheduler.toml) - for scheduled triggers +- [plugin_config_data_writes.toml](plugin_config_data_writes.toml) - for data write triggers + +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](/README.md). + +## [Special Requirements Section] + + + +### Data requirements + +The plugin requires [specific data format or schema requirements]. + +### Software requirements + +- **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled +- **Python packages**: + - `package_name` (for specific functionality) + +## Installation steps + +1. Start InfluxDB 3 with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` + +2. Install required Python packages (if any): + + ```bash + influxdb3 install package package_name + ``` + +## Trigger setup + +### Scheduled trigger + +Run the plugin periodically on historical data: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "every:1h" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + scheduled_trigger_name +``` + +### Data write trigger + +Process data as it's written: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "all_tables" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + write_trigger_name +``` + +### HTTP trigger + +Process data via HTTP requests: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "request:endpoint" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + http_trigger_name +``` + +## Example usage + +### Example 1: [Use case name] + +[Description of what this example demonstrates] + +```bash +# Create the trigger +influxdb3 create trigger \ + --database weather \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "every:30m" \ + --trigger-arguments 'parameter_name=value,another_param=100' \ + example_trigger + +# Write test data +influxdb3 write \ + --database weather \ + "measurement,tag=value field=22.5" + +# Query results (after trigger runs) +influxdb3 query \ + --database weather \ + "SELECT * FROM result_measurement" +``` + +### Expected output + +``` +tag | field | time +----|-------|----- +value | 22.5 | 2024-01-01T00:00:00Z +``` + +**Transformation details:** +- Before: `field=22.5` (original value) +- After: `field=22.5` (processed value with description of changes) + +### Example 2: [Another use case] + +[Description of what this example demonstrates] + +```bash +# Create trigger with different configuration +influxdb3 create trigger \ + --database sensors \ + --plugin-filename gh:influxdata/plugin_name/plugin_name.py \ + --trigger-spec "all_tables" \ + --trigger-arguments 'parameter_name=different_value,optional_param=true' \ + another_trigger + +# Write data with specific format +influxdb3 write \ + --database sensors \ + "raw_data,device=sensor1 value1=20.1,value2=45.2" + +# Query processed data +influxdb3 query \ + --database sensors \ + "SELECT * FROM processed_data" +``` + +### Expected output + +``` +device | value1 | value2 | time +-------|--------|--------|----- +sensor1 | 20.1 | 45.2 | 2024-01-01T00:00:00Z +``` + +**Processing details:** +- Before: `value1=20.1`, `value2=45.2` +- After: [Description of any transformations applied] + +### Example 3: [Complex scenario] + +[Add more examples as needed to demonstrate different features] + +## Code overview + +### Files + +- `plugin_name.py`: Main plugin code containing handlers for trigger types +- `plugin_config_scheduler.toml`: Example TOML configuration for scheduled triggers +- `plugin_config_data_writes.toml`: Example TOML configuration for data write triggers + +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` +Handles scheduled trigger execution. Queries historical data within the specified window and applies processing logic. + +Key operations: +1. Parses configuration from arguments +2. Queries source data with filters +3. Applies processing logic +4. Writes results to target measurement + +#### `process_writes(influxdb3_local, table_batches, args)` +Handles real-time data processing during writes. Processes incoming data batches and applies transformations before writing. + +Key operations: +1. Filters relevant table batches +2. Applies processing to each row +3. Writes to target measurement immediately + +#### `process_http_request(influxdb3_local, request_body, args)` +Handles HTTP-triggered processing. Processes data sent via HTTP requests. + +Key operations: +1. Parses request body +2. Validates input data +3. Applies processing logic +4. Returns response + +### Key logic + +1. **Data Validation**: Checks input data format and required fields +2. **Processing**: Applies the main plugin logic to transform/analyze data +3. **Output Generation**: Formats results and metadata +4. **Error Handling**: Manages exceptions and provides meaningful error messages + +### Plugin Architecture + +``` +Plugin Module +ā”œā”€ā”€ process_scheduled_call() # Scheduled trigger handler +ā”œā”€ā”€ process_writes() # Data write trigger handler +ā”œā”€ā”€ process_http_request() # HTTP trigger handler +ā”œā”€ā”€ validate_config() # Configuration validation +ā”œā”€ā”€ apply_processing() # Core processing logic +ā””ā”€ā”€ helper_functions() # Utility functions +``` + +## Troubleshooting + +### Common issues + +#### Issue: "Configuration parameter missing" +**Solution**: Check that all required parameters are provided in the trigger arguments. Verify parameter names match exactly (case-sensitive). + +#### Issue: "Permission denied" errors +**Solution**: Ensure the plugin file has execute permissions: +```bash +chmod +x ~/.plugins/plugin_name.py +``` + +#### Issue: "Module not found" error +**Solution**: Install required Python packages: +```bash +influxdb3 install package package_name +``` + +#### Issue: No data in target measurement +**Solution**: +1. Check that source measurement contains data +2. Verify trigger is enabled and running +3. Check logs for errors: + ```bash + influxdb3 query \ + --database _internal \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" + ``` + +### Debugging tips + +1. **Enable debug logging**: Add `debug=true` to trigger arguments +2. **Use dry run mode**: Set `dry_run=true` to test without writing data +3. **Check field names**: Use `SHOW FIELD KEYS FROM measurement` to verify field names +4. **Test with small windows**: Use short time windows for testing (e.g., `window=1h`) +5. **Monitor resource usage**: Check CPU and memory usage during processing + +### Performance considerations + +- Processing large datasets may require increased memory +- Use filters to process only relevant data +- Batch size affects memory usage and processing speed +- Consider using specific_fields to limit processing scope +- Cache frequently accessed data when possible + +## Questions/Comments + +For questions or comments about this plugin, please open an issue in the [influxdb3_plugins repository](https://github.com/influxdata/influxdb3_plugins/issues). + +--- + +## Documentation Sync + +After making changes to this README, sync to the documentation site: + +1. **Commit your changes** to the influxdb3_plugins repository +2. **Look for the sync reminder** - A comment will appear on your commit with a sync link +3. **Click the link** - This opens a pre-filled form to trigger the docs-v2 sync +4. **Submit the sync request** - A workflow will validate, transform, and create a PR + +The documentation site will be automatically updated with your changes after review. + +--- + +## Template Usage Notes + +When using this template: + +1. Replace `Plugin Name` with the actual plugin name +2. Update emoji metadata with appropriate trigger types and tags +3. Fill in all parameter tables with actual configuration options +4. Provide real, working examples with expected output +5. Include actual function names and signatures +6. Add plugin-specific troubleshooting scenarios +7. Remove any sections that don't apply to your plugin +8. Remove this "Template Usage Notes" section from the final README + +### Section Guidelines + +- **Description**: 2-4 sentences, be specific about capabilities +- **Configuration**: Group parameters logically, mark required clearly +- **Examples**: At least 2 complete, working examples +- **Expected output**: Show actual output format +- **Troubleshooting**: Include plugin-specific issues and solutions +- **Code overview**: Document main functions and logic flow \ No newline at end of file diff --git a/influxdata/basic_transformation/README.md b/influxdata/basic_transformation/README.md index a34bb4b..98cdb6e 100644 --- a/influxdata/basic_transformation/README.md +++ b/influxdata/basic_transformation/README.md @@ -56,7 +56,7 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor - [basic_transformation_config_scheduler.toml](basic_transformation_config_scheduler.toml) - for scheduled triggers - [basic_transformation_config_data_writes.toml](basic_transformation_config_data_writes.toml) - for data write triggers -For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](/README.md). +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](../../README.md). ## Data requirements @@ -68,7 +68,7 @@ The plugin assumes that the table schema is already defined in the database, as - **Python packages**: - `pint` (for unit conversions) -### Installation steps +## Installation steps 1. Start InfluxDB 3 with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): diff --git a/scripts/validate_readme.py b/scripts/validate_readme.py new file mode 100644 index 0000000..ff6a731 --- /dev/null +++ b/scripts/validate_readme.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python3 +""" +Validates plugin README files against the standard template. +Ensures consistency across all plugin documentation. +""" + +import sys +import re +import argparse +from pathlib import Path +from typing import List, Tuple + +# Validation thresholds and limits +MINIMUM_BASH_EXAMPLES = 2 +MINIMUM_EXPECTED_OUTPUT_SECTIONS = 1 +MINIMUM_TROUBLESHOOTING_ISSUES = 2 +SUMMARY_SEPARATOR_LENGTH = 60 + +# Section search offsets +SECTION_SEARCH_OFFSET = 100 +SECTION_NOT_FOUND = -1 + +# Exit codes +EXIT_SUCCESS = 0 +EXIT_ERROR = 1 +EMOJI_METADATA_PATTERN = r'^āš”\s+[\w\-,\s]+\s+šŸ·ļø\s+[\w\-,\s]+\s+šŸ”§\s+InfluxDB 3' + +# Required sections in order +REQUIRED_SECTIONS = [ + ("# ", "Title with plugin name"), + ("## Description", "Plugin description"), + ("## Configuration", "Configuration overview"), + ("### Plugin metadata", "Metadata description"), + ("### Required parameters", "Required parameters table"), + ("## Installation steps", "Installation instructions"), + ("## Trigger setup", "Trigger configuration examples"), + ("## Example usage", "Usage examples"), + ("## Code overview", "Code structure description"), + ("## Troubleshooting", "Common issues and solutions"), + ("## Questions/Comments", "Support information") +] + +# Optional but recommended sections +OPTIONAL_SECTIONS = [ + "### Optional parameters", + "### TOML configuration", + "### Data requirements", + "### Software requirements", + "### Schema requirements", + "### Debugging tips", + "### Performance considerations" +] + +OPTIONAL_SECTIONS_SKIP_WARNING = ["### Debugging tips", "### Performance considerations"] + +def extract_section_content(content: str, section_heading: str) -> str: + """ + Extract content between a markdown section heading and the next same-level heading. + + This function properly handles: + - Subsections (###, ####) within the section + - Code blocks containing ## characters + - Section being at start or end of document + - Missing sections + + Args: + content: Full markdown document content + section_heading: Heading text without the ## prefix (e.g., "Troubleshooting") + + Returns: + Section content as string, or empty string if section not found + + Examples: + >>> extract_section_content(doc, "Troubleshooting") + '### Common issues\\n#### Issue: ...' + """ + # Build the section marker - look for "## Heading" where Heading doesn't start with # + # This ensures we match "## Troubleshooting" but not "### Troubleshooting" + section_pattern = f'## {section_heading}' + + # Find the section heading + section_index = content.find(section_pattern) + if section_index == -1: + return "" + + # Find the end of the heading line (start of content) + content_start = content.find('\n', section_index) + if content_start == -1: + # Section heading is at end of file with no content + return "" + content_start += 1 # Move past the newline + + # Find the next same-level heading (## followed by space, not ###) + # Use a loop to ensure we're not matching subsections + search_pos = content_start + while True: + next_heading_pos = content.find('\n## ', search_pos) + + if next_heading_pos == -1: + # No more headings, take everything to end of document + return content[content_start:].rstrip() + + # Check that it's actually a ## heading and not ### + # Look at the character after '## ' + check_pos = next_heading_pos + 4 # Position after '\n## ' + if check_pos < len(content) and content[check_pos] != '#': + # This is a proper ## heading, not ### or #### + return content[content_start:next_heading_pos].rstrip() + + # This was a false match (like \n### ), keep searching + search_pos = next_heading_pos + 1 + + # Safety: if we've searched too far, something is wrong + if search_pos > len(content): + return content[content_start:].rstrip() + +def validate_emoji_metadata(content: str) -> List[str]: + """Validate the emoji metadata line.""" + errors = [] + + # Check for emoji metadata pattern + if not re.search(EMOJI_METADATA_PATTERN, content, re.MULTILINE): + errors.append("Missing or invalid emoji metadata line (should have āš” trigger types šŸ·ļø tags šŸ”§ compatibility)") + + return errors + +def validate_sections(content: str) -> List[str]: + """Validate required sections are present and in order.""" + errors = [] + lines = content.split('\n') + + # Track section positions + section_positions = {} + for i, line in enumerate(lines): + for section, description in REQUIRED_SECTIONS: + if line.startswith(section) and section not in section_positions: + # Special handling for title (should contain actual plugin name) + if section == "# " and not line.startswith("# Plugin Name"): + section_positions[section] = i + elif section != "# ": + section_positions[section] = i + + # Check all required sections are present + for section, description in REQUIRED_SECTIONS: + if section not in section_positions: + errors.append(f"Missing required section: '{section.strip()}' - {description}") + + # Check sections are in correct order + if len(section_positions) == len(REQUIRED_SECTIONS): + positions = list(section_positions.values()) + if positions != sorted(positions): + errors.append("Sections are not in the correct order (see template for proper ordering)") + + return errors + +def validate_parameter_tables(content: str) -> List[str]: + """Validate parameter table formatting.""" + errors = [] + + # Check for parameter table headers with flexible whitespace + # This regex allows variable spacing between columns + table_pattern = r'\|\s*Parameter\s*\|\s*Type\s*\|\s*Default\s*\|\s*Description\s*\|' + if not re.search(table_pattern, content): + errors.append("No properly formatted parameter tables found (should have Parameter | Type | Default | Description columns)") + + # Validate Required parameters section if present + if '### Required parameters' in content: + # Find the subsection + subsection_start = content.find('### Required parameters') + + # Find the end: next subsection (###) or next section (##) + # Look for '\n### ' or '\n## ' after current position + search_pos = subsection_start + len('### Required parameters') + + next_subsection = content.find('\n### ', search_pos) + next_section = content.find('\n## ', search_pos) + + # Determine the end position + if next_subsection == -1 and next_section == -1: + subsection_end = len(content) + elif next_subsection == -1: + subsection_end = next_section + elif next_section == -1: + subsection_end = next_subsection + else: + subsection_end = min(next_subsection, next_section) + + section_content = content[subsection_start:subsection_end] + + # Check that it indicates which parameters are required + if 'required' not in section_content.lower() and 'yes' not in section_content.lower(): + errors.append("Required parameters section should indicate which parameters are required") + + return errors + +def validate_examples(content: str) -> List[str]: + """Validate code examples and expected output.""" + errors = [] + + # Check for bash code examples + bash_examples = re.findall(r'```bash(.*?)```', content, re.DOTALL) + if len(bash_examples) < MINIMUM_BASH_EXAMPLES: + errors.append(f"Should have at least {MINIMUM_BASH_EXAMPLES} bash code examples (found {len(bash_examples)})") + + # Check for influxdb3 commands in examples + has_create_trigger = any('influxdb3 create trigger' in ex for ex in bash_examples) + has_write_data = any('influxdb3 write' in ex for ex in bash_examples) + has_query = any('influxdb3 query' in ex for ex in bash_examples) + + if not has_create_trigger: + errors.append("Examples should include 'influxdb3 create trigger' command") + if not has_write_data: + errors.append("Examples should include 'influxdb3 write' command for test data") + if not has_query: + errors.append("Examples should include 'influxdb3 query' command to verify results") + + # Check for expected output + expected_output_count = content.count('### Expected output') + content.count('**Expected output') + if expected_output_count < MINIMUM_EXPECTED_OUTPUT_SECTIONS: + errors.append(f"Should include at least {MINIMUM_EXPECTED_OUTPUT_SECTIONS} 'Expected output' section in examples") + + return errors + +def validate_links(content: str, plugin_path: Path) -> List[str]: + """Validate internal links and references.""" + errors = [] + + # Check for TOML file references if TOML configuration is mentioned + if '### TOML configuration' in content: + toml_links = re.findall(r'\[([^\]]+\.toml)\]\(([^)]+)\)', content) + plugin_dir = plugin_path.parent + + for link_text, link_path in toml_links: + # Check if it's a relative link (not starting with http) + if not link_path.startswith('http'): + toml_file = plugin_dir / link_path + if not toml_file.exists(): + errors.append(f"Referenced TOML file not found: {link_path}") + + # Check for influxdb3_plugins README reference + if '/README.md' in content and 'influxdb3_plugins/README.md' not in content: + errors.append("Link to main README should reference 'influxdb3_plugins/README.md'") + + return errors + +def validate_troubleshooting(content: str) -> List[str]: + """Validate troubleshooting section content.""" + errors = [] + + # Check if section exists + if '## Troubleshooting' not in content: + return errors + + # Extract the section content using robust extraction + section_content = extract_section_content(content, 'Troubleshooting') + + # Defensive check + if not section_content.strip(): + errors.append("Troubleshooting section exists but appears empty") + return errors + + # Check for required subsections + if '### Common issues' not in section_content: + errors.append("Troubleshooting should include 'Common issues' subsection") + + # Check for issue/solution patterns + # Count both markdown heading style (####) and bold style (**) + issue_count = section_content.count('#### Issue:') + section_content.count('**Issue:') + solution_count = (section_content.count('**Solution**:') + section_content.count('**Solution:') + section_content.count('Solution:')) + + if issue_count < MINIMUM_TROUBLESHOOTING_ISSUES: + errors.append(f"Troubleshooting should include at least {MINIMUM_TROUBLESHOOTING_ISSUES} documented issues") + + if issue_count > 0 and solution_count < issue_count: + errors.append("Each troubleshooting issue should have a corresponding solution") + + return errors + +def validate_code_overview(content: str) -> List[str]: + """Validate code overview section.""" + errors = [] + + # Check if section exists + if '## Code overview' not in content: + return errors + + # Extract the section content using robust extraction + section_content = extract_section_content(content, 'Code overview') + + # Defensive check + if not section_content.strip(): + errors.append("Code overview section exists but appears empty") + return errors + + # Check for required subsections + if '### Files' not in section_content: + errors.append("Code overview should include 'Files' subsection") + + if not ('### Main functions' in section_content or '### Key functions' in section_content): + errors.append("Code overview should include 'Main functions' or 'Key functions' subsection") + + # Check for function documentation (signatures in backticks or Python def) + has_function_signatures = ( + 'def ' in section_content or + re.search(r'####?\s+`\w+\(.*?\)`', section_content) or + re.search(r'`\w+\([^)]*\)`:', section_content) + ) + + if not has_function_signatures: + errors.append("Code overview should document main functions with their signatures") + + return errors + +def format_validation_result(readme_path: Path, errors: List[str], warnings: List[str]) -> str: + """Format validation results for display.""" + result = [] + + if not errors and not warnings: + result.append(f"āœ… {readme_path}") + else: + result.append(f"\n{'āŒ' if errors else 'āš ļø'} {readme_path}:") + + if errors: + result.append(" Errors:") + for error in errors: + result.append(f" - {error}") + + if warnings: + result.append(" Warnings:") + for warning in warnings: + result.append(f" - {warning}") + + return '\n'.join(result) + +def validate_readme(readme_path: Path) -> Tuple[List[str], List[str]]: + """ + Validate a single README file. + Returns tuple of (errors, warnings). + """ + try: + with open(readme_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception as e: + return [f"Could not read file: {e}"], [] + + errors = [] + warnings = [] + + # Run all validations + errors.extend(validate_emoji_metadata(content)) + errors.extend(validate_sections(content)) + errors.extend(validate_parameter_tables(content)) + errors.extend(validate_examples(content)) + errors.extend(validate_links(content, readme_path)) + errors.extend(validate_troubleshooting(content)) + errors.extend(validate_code_overview(content)) + + # Check for optional but recommended sections + for section in OPTIONAL_SECTIONS: + if section not in content and section not in OPTIONAL_SECTIONS_SKIP_WARNING: + warnings.append(f"Consider adding '{section}' section") + + # Check for template remnants + if 'Plugin Name' in content and '# Plugin Name' in content: + errors.append("README still contains template placeholder 'Plugin Name'") + if 'Template Usage Notes' in content: + errors.append("README still contains 'Template Usage Notes' section (should be removed)") + + return errors, warnings + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description='Validates plugin README files against the standard template.', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +Examples: + python scripts/validate_readme.py # Validate all plugins + python scripts/validate_readme.py --plugins basic_transformation,downsampler + python scripts/validate_readme.py --list # List available plugins + python scripts/validate_readme.py --quiet # Show only errors + +Validation Rules: + - Checks for required sections in correct order + - Validates emoji metadata format + - Ensures parameter tables are properly formatted + - Verifies code examples include required commands + - Validates troubleshooting content structure + ''' + ) + + parser.add_argument( + '--plugins', + type=str, + help='Comma-separated list of specific plugins to validate (e.g., "basic_transformation,downsampler")' + ) + + parser.add_argument( + '--list', + action='store_true', + help='List all available plugins and exit' + ) + + parser.add_argument( + '--quiet', + action='store_true', + help='Show only errors, suppress warnings and success messages' + ) + + parser.add_argument( + '--errors-only', + action='store_true', + help='Exit with success code even if warnings are found (only fail on errors)' + ) + + return parser.parse_args() + +def list_available_plugins(): + """List all available plugins and exit.""" + influxdata_dir = Path('influxdata') + + if not influxdata_dir.exists(): + print("āŒ Error: 'influxdata' directory not found. Run this script from the influxdb3_plugins root directory.") + sys.exit(EXIT_ERROR) + + readme_files = list(influxdata_dir.glob('*/README.md')) + + if not readme_files: + print("āŒ No plugins found in influxdata/ subdirectories") + sys.exit(EXIT_ERROR) + + print(f"Available plugins ({len(readme_files)} found):") + for readme_path in sorted(readme_files): + plugin_name = readme_path.parent.name + print(f" - {plugin_name}") + + sys.exit(EXIT_SUCCESS) + +def filter_plugins_by_name(readme_files: List[Path], plugin_names: str) -> List[Path]: + """Filter README files by specified plugin names.""" + requested_plugins = [name.strip() for name in plugin_names.split(',')] + filtered_files = [] + + for readme_path in readme_files: + plugin_name = readme_path.parent.name + if plugin_name in requested_plugins: + filtered_files.append(readme_path) + requested_plugins.remove(plugin_name) + + # Report any plugins that weren't found + if requested_plugins: + print(f"āš ļø Warning: The following plugins were not found: {', '.join(requested_plugins)}") + available_plugins = [f.parent.name for f in readme_files] + print(f"Available plugins: {', '.join(sorted(available_plugins))}") + + return filtered_files + +def main(): + """Main validation function.""" + args = parse_arguments() + + # Handle list option + if args.list: + list_available_plugins() + + # Find all plugin READMEs + influxdata_dir = Path('influxdata') + + if not influxdata_dir.exists(): + print("āŒ Error: 'influxdata' directory not found. Run this script from the influxdb3_plugins root directory.") + sys.exit(EXIT_ERROR) + + readme_files = list(influxdata_dir.glob('*/README.md')) + + if not readme_files: + print("āŒ No README files found in influxdata/ subdirectories") + sys.exit(EXIT_ERROR) + + # Filter by specific plugins if requested + if args.plugins: + readme_files = filter_plugins_by_name(readme_files, args.plugins) + if not readme_files: + print("āŒ No matching plugins found") + sys.exit(EXIT_ERROR) + + if not args.quiet: + print(f"Validating {len(readme_files)} plugin README files...\n") + + all_valid = True + error_count = 0 + warning_count = 0 + + for readme_path in sorted(readme_files): + errors, warnings = validate_readme(readme_path) + + if errors: + all_valid = False + error_count += len(errors) + warning_count += len(warnings) + + result = format_validation_result(readme_path, errors, warnings) + + # Apply quiet mode filtering + if args.quiet: + # Only show files with errors in quiet mode + if errors: + print(result) + else: + print(result) + + # Print summary + if not args.quiet: + print("\n" + "=" * SUMMARY_SEPARATOR_LENGTH) + print("VALIDATION SUMMARY") + print("=" * SUMMARY_SEPARATOR_LENGTH) + print(f"Total files validated: {len(readme_files)}") + print(f"Errors found: {error_count}") + print(f"Warnings found: {warning_count}") + + # Determine exit status + has_errors = error_count > 0 + has_warnings = warning_count > 0 + + if not has_errors and not has_warnings: + if not args.quiet: + print("\nāœ… All README files are valid!") + sys.exit(EXIT_SUCCESS) + elif not has_errors and has_warnings: + if not args.quiet: + print(f"\nāš ļø Validation completed with {warning_count} warning(s) but no errors") + # If --errors-only is specified, exit successfully even with warnings + exit_code = EXIT_SUCCESS if args.errors_only else EXIT_ERROR + sys.exit(exit_code) + else: + if not args.quiet: + print(f"\nāŒ Validation failed with {error_count} error(s)") + if has_warnings: + print(f"Also found {warning_count} warning(s)") + print("\nPlease fix the errors above and ensure all READMEs follow the template.") + print("See README_TEMPLATE.md for the correct structure.") + sys.exit(EXIT_ERROR) + +if __name__ == "__main__": + main() \ No newline at end of file