Testing Handbook Generator

# Validate all generated skills
uv run scripts/validate-skills.py

# Validate specific skill
uv run scripts/validate-skills.py --skill libfuzzer

# Output JSON for CI integration
uv run scripts/validate-skills.py --json

# Verbose output with details
uv run scripts/validate-skills.py -v

uv run scripts/validate-skills.py --skill [skill-name]

# Validate all skills at once
uv run scripts/validate-skills.py

# JSON output for CI pipelines
uv run scripts/validate-skills.py --json

SKILL="../libfuzzer/SKILL.md" && \
  [ $(wc -l < "$SKILL") -lt 500 ] && \
  [ $(grep -cE '\{\{[<%]' "$SKILL") -eq 0 ] && \
  awk '/^---$/{if(++n==2)exit}n==1' "$SKILL" | yq -e '.' >/dev/null 2>&1 && \
  echo "✓ Valid: $SKILL" || echo "✗ Invalid: $SKILL"

# Set skill path (replace 'libfuzzer' with target skill name)
SKILL="../libfuzzer/SKILL.md"

# Extract and validate frontmatter (awk extracts content between first two ---)
awk '/^---$/{if(++n==2)exit}n==1' "$SKILL" | yq -e '.'

# Validate name field
NAME=$(yq '.name' "$SKILL")
echo "$NAME" | grep -qE '^[a-z0-9-]{1,64}$' && echo "Name: OK" || echo "Name: INVALID"

# Validate description length
DESC=$(yq '.description' "$SKILL")
[ ${#DESC} -le 1024 ] && [ ${#DESC} -gt 0 ] && echo "Description: OK" || echo "Description: INVALID"

# Check for trigger phrase in description
DESC=$(yq '.description' "$SKILL")
echo "$DESC" | grep -qE 'Use when|Use for' && echo "Trigger: OK" || echo "Trigger: MISSING"

# Validate required sections for a tool skill
SKILL="../semgrep/SKILL.md"
REQUIRED=("When to Use" "Quick Reference" "Installation" "Core Workflow")

for section in "${REQUIRED[@]}"; do
  grep -q "^## $section" "$SKILL" && echo "✓ $section" || echo "✗ Missing: $section"
done

# Check Related Skills section exists
grep -q "^## Related Skills" "$SKILL" || echo "✗ Missing: Related Skills section"

# Extract skill references and check they exist
grep -oE '\*\*[a-z0-9-]+\*\*' "$SKILL" | tr -d '*' | sort -u | while read ref; do
  [ -d "../$ref" ] && echo "✓ $ref exists" || echo "⚠ $ref not found (may be planned)"
done

wc -l ../libfuzzer/SKILL.md  # Replace 'libfuzzer' with target skill
# Should be < 500

SKILL="../libfuzzer/SKILL.md"  # Replace 'libfuzzer' with target skill
SKILL_DIR=$(dirname "$SKILL")

# Find markdown links and verify they exist
grep -oE '\[.+\]\(([^)]+)\)' "$SKILL" | grep -oE '\([^)]+\)' | tr -d '()' | while read link; do
  # Skip external URLs
  [[ "$link" =~ ^https?:// ]] && continue
  # Check if file exists
  [ -f "$SKILL_DIR/$link" ] && echo "✓ $link" || echo "✗ $link (not found)"
done

# Should return nothing (exit code 1 = pass, exit code 0 = shortcodes found)
grep -E '\{\{[<%]' ../libfuzzer/SKILL.md  # Replace 'libfuzzer' with target skill

┌─────────────────────────────────────────────────────┐
│                    Generate Skill                    │
└─────────────────────────────────────────────────────┘
                         │
                         ▼
┌─────────────────────────────────────────────────────┐
│         Run Validator Script                         │
│  uv run scripts/validate-skills.py --skill [name]   │
│                                                     │
│  Checks: YAML, sections, line count, shortcodes,   │
│          cross-refs, internal links                 │
└─────────────────────────────────────────────────────┘
                         │
              ┌──────────┴──────────┐
              │                     │
         All Pass              Failed
              │                     │
              ▼                     ▼
┌─────────────────────┐   ┌─────────────────────┐
│  Activation Test    │   │   Fix Issues        │
│  - Direct invoke    │   │   - Edit skill      │
│  - Implicit trigger │   │   - Re-run validator│
│  - Decision tree    │   └─────────────────────┘
└─────────────────────┘            │
              │                    │
              ▼                    │
┌─────────────────────┐            │
│     Pass?           │            │
└─────────────────────┘            │
         │                         │
    ┌────┴────┐                    │
   Yes        No                   │
    │         │                    │
    │         └────────────────────┘
    ▼
┌─────────────────────────────────────────────────────┐
│                  Mark Complete                       │
│  - Report success to user                           │
│  - List any warnings from validator                 │
└─────────────────────────────────────────────────────┘

## Skill: [name]

### Automated Validation
- [ ] Validator passes: `uv run scripts/validate-skills.py --skill [name]`
- [ ] No errors in output
- [ ] Warnings reviewed and addressed (or documented as acceptable)

### Content Quality (manual review)
- [ ] Description leads with the task, names concrete anchors, and gives two or more situations
- [ ] Every command, flag, file name, and API symbol the description advertises appears in the skill body
- [ ] When to Use section has clear triggers
- [ ] Quick Reference is actionable
- [ ] Code examples are complete and runnable
- [ ] Resources section has titles and URLs

### Activation Testing
- [ ] Direct invocation works
- [ ] Implicit trigger matches expected prompts
- [ ] Decision tree navigation correct

# Generate JSON report
uv run scripts/validate-skills.py --json > validation-report.json

# Or view in terminal
uv run scripts/validate-skills.py

==================================================
VALIDATION REPORT
==================================================
✓ libfuzzer
✓ semgrep ⚠
  WARNING: Line count 480 approaching limit of 500
✗ wycheproof
  ERROR: Missing required sections for domain skill: ['Background']

--------------------------------------------------
Total:    3
Passed:   2
Failed:   1
Warnings: 1
--------------------------------------------------
✗ 1 skill(s) failed validation

# Skill Generation Report

## Validator Results
- Total: X
- Passed: Y
- Failed: Z
- With warnings: W

## Actions Needed

### Failed Skills
1. **wycheproof**: Add Background section

### Warnings
1. **semgrep**: Line count 480 - consider splitting

## Next Steps
- [ ] Fix failed skills
- [ ] Address warnings
- [ ] Run activation tests

## Self-Improvement Log

### YYYY-MM-DD: {Brief title}

**Issue:** {What went wrong during generation}
**Example:** {Specific skill/section affected}
**Root cause:** {Why the current logic failed}
**Fix:** {What was changed}
**Files modified:**
- `{file1}`: {what changed}
- `{file2}`: {what changed}
**Verification:** {How to confirm the fix works}