📦 deps(thirdparty): update snapshots
This commit is contained in:
@@ -115,17 +115,12 @@ Full workflow execution test (~10-30 minutes):
|
||||
- Subagents follow the skill correctly
|
||||
- Final code is functional and tested
|
||||
|
||||
#### test-requesting-code-review.sh
|
||||
Behavioral test for the code reviewer subagent (~5 minutes):
|
||||
- Builds a tiny project with a baseline commit
|
||||
- Adds a second commit that plants two real bugs (SQL injection, plaintext password handling)
|
||||
- Dispatches the code reviewer via the requesting-code-review skill
|
||||
- Verifies the reviewer flags the planted bugs at Critical/Important severity and refuses to approve
|
||||
|
||||
**What it tests:**
|
||||
- The skill actually dispatches a working code reviewer subagent
|
||||
- The reviewer template produces reviewers that catch obvious security bugs
|
||||
- The reviewer is not sycophantic — it does not approve a diff with planted Critical issues
|
||||
#### test-worktree-native-preference.sh
|
||||
RED-GREEN-REFACTOR validation for the using-git-worktrees skill (~5 minutes):
|
||||
- RED: skill without Step 1a — agent should use `git worktree add`
|
||||
- GREEN: skill with Step 1a — agent should use the native EnterWorktree tool
|
||||
- PRESSURE: same as GREEN under urgency framing with pre-existing `.worktrees/`
|
||||
- Drill scenario `worktree-creation-under-pressure.yaml` covers the PRESSURE phase only
|
||||
|
||||
## Adding New Tests
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ fi
|
||||
# Parse command line arguments
|
||||
VERBOSE=false
|
||||
SPECIFIC_TEST=""
|
||||
TIMEOUT=300 # Default 5 minute timeout per test
|
||||
TIMEOUT=600 # Default 10 minute timeout per test
|
||||
RUN_INTEGRATION=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -73,13 +73,13 @@ done
|
||||
|
||||
# List of skill tests to run (fast unit tests)
|
||||
tests=(
|
||||
"test-worktree-path-policy.sh"
|
||||
"test-subagent-driven-development.sh"
|
||||
)
|
||||
|
||||
# Integration tests (slow, full execution)
|
||||
integration_tests=(
|
||||
"test-subagent-driven-development-integration.sh"
|
||||
"test-requesting-code-review.sh"
|
||||
)
|
||||
|
||||
# Add integration tests if requested
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Integration Test: Document Review System
|
||||
# Actually runs spec/plan review and verifies reviewers catch issues
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/test-helpers.sh"
|
||||
|
||||
echo "========================================"
|
||||
echo " Integration Test: Document Review System"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "This test verifies the document review system by:"
|
||||
echo " 1. Creating a spec with intentional errors"
|
||||
echo " 2. Running the spec document reviewer"
|
||||
echo " 3. Verifying the reviewer catches the errors"
|
||||
echo ""
|
||||
|
||||
# Create test project
|
||||
TEST_PROJECT=$(create_test_project)
|
||||
echo "Test project: $TEST_PROJECT"
|
||||
|
||||
# Trap to cleanup
|
||||
trap "cleanup_test_project $TEST_PROJECT" EXIT
|
||||
|
||||
cd "$TEST_PROJECT"
|
||||
|
||||
# Create directory structure
|
||||
mkdir -p docs/superpowers/specs
|
||||
|
||||
# Create a spec document WITH INTENTIONAL ERRORS for the reviewer to catch
|
||||
cat > docs/superpowers/specs/test-feature-design.md <<'EOF'
|
||||
# Test Feature Design
|
||||
|
||||
## Overview
|
||||
|
||||
This is a test feature that does something useful.
|
||||
|
||||
## Requirements
|
||||
|
||||
1. The feature should work correctly
|
||||
2. It should be fast
|
||||
3. TODO: Add more requirements here
|
||||
|
||||
## Architecture
|
||||
|
||||
The feature will use a simple architecture with:
|
||||
- A frontend component
|
||||
- A backend service
|
||||
- Error handling will be specified later once we understand the failure modes better
|
||||
|
||||
## Data Flow
|
||||
|
||||
Data flows from the frontend to the backend.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
Tests will be written to cover the main functionality.
|
||||
EOF
|
||||
|
||||
# Initialize git repo
|
||||
git init --quiet
|
||||
git config user.email "test@test.com"
|
||||
git config user.name "Test User"
|
||||
git add .
|
||||
git commit -m "Initial commit with test spec" --quiet
|
||||
|
||||
echo ""
|
||||
echo "Created test spec with intentional errors:"
|
||||
echo " - TODO placeholder in Requirements section"
|
||||
echo " - 'specified later' deferral in Architecture section"
|
||||
echo ""
|
||||
echo "Running spec document reviewer..."
|
||||
echo ""
|
||||
|
||||
# Run Claude to review the spec
|
||||
OUTPUT_FILE="$TEST_PROJECT/claude-output.txt"
|
||||
|
||||
PROMPT="You are testing the spec document reviewer.
|
||||
|
||||
Read the spec-document-reviewer-prompt.md template in skills/brainstorming/ to understand the review format.
|
||||
|
||||
Then review the spec at $TEST_PROJECT/docs/superpowers/specs/test-feature-design.md using the criteria from that template.
|
||||
|
||||
Look for:
|
||||
- TODOs, placeholders, 'TBD', incomplete sections
|
||||
- Sections saying 'to be defined later' or 'will spec when X is done'
|
||||
- Sections noticeably less detailed than others
|
||||
|
||||
Output your review in the format specified in the template."
|
||||
|
||||
echo "================================================================================"
|
||||
cd "$SCRIPT_DIR/../.." && timeout 120 claude -p "$PROMPT" --permission-mode bypassPermissions 2>&1 | tee "$OUTPUT_FILE" || {
|
||||
echo ""
|
||||
echo "================================================================================"
|
||||
echo "EXECUTION FAILED (exit code: $?)"
|
||||
exit 1
|
||||
}
|
||||
echo "================================================================================"
|
||||
|
||||
echo ""
|
||||
echo "Analyzing reviewer output..."
|
||||
echo ""
|
||||
|
||||
# Verification tests
|
||||
FAILED=0
|
||||
|
||||
echo "=== Verification Tests ==="
|
||||
echo ""
|
||||
|
||||
# Test 1: Reviewer found the TODO
|
||||
echo "Test 1: Reviewer found TODO..."
|
||||
if grep -qi "TODO" "$OUTPUT_FILE" && grep -qi "requirements\|Requirements" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Reviewer identified TODO in Requirements section"
|
||||
else
|
||||
echo " [FAIL] Reviewer did not identify TODO"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 2: Reviewer found the "specified later" deferral
|
||||
echo "Test 2: Reviewer found 'specified later' deferral..."
|
||||
if grep -qi "specified later\|later\|defer\|incomplete\|error handling" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Reviewer identified deferred content"
|
||||
else
|
||||
echo " [FAIL] Reviewer did not identify deferred content"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 3: Reviewer output includes Issues section
|
||||
echo "Test 3: Review output format..."
|
||||
if grep -qi "issues\|Issues" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Review includes Issues section"
|
||||
else
|
||||
echo " [FAIL] Review missing Issues section"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 4: Reviewer did NOT approve (found issues)
|
||||
echo "Test 4: Reviewer verdict..."
|
||||
if grep -qi "Issues Found\|❌\|not approved\|issues found" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Reviewer correctly found issues (not approved)"
|
||||
elif grep -qi "Approved\|✅" "$OUTPUT_FILE" && ! grep -qi "Issues Found\|❌" "$OUTPUT_FILE"; then
|
||||
echo " [FAIL] Reviewer incorrectly approved spec with errors"
|
||||
FAILED=$((FAILED + 1))
|
||||
else
|
||||
echo " [PASS] Reviewer identified problems (ambiguous format but found issues)"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Summary
|
||||
echo "========================================"
|
||||
echo " Test Summary"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo "STATUS: PASSED"
|
||||
echo "All verification tests passed!"
|
||||
echo ""
|
||||
echo "The spec document reviewer correctly:"
|
||||
echo " ✓ Found TODO placeholder"
|
||||
echo " ✓ Found 'specified later' deferral"
|
||||
echo " ✓ Produced properly formatted review"
|
||||
echo " ✓ Did not approve spec with errors"
|
||||
exit 0
|
||||
else
|
||||
echo "STATUS: FAILED"
|
||||
echo "Failed $FAILED verification tests"
|
||||
echo ""
|
||||
echo "Output saved to: $OUTPUT_FILE"
|
||||
echo ""
|
||||
echo "Review the output to see what went wrong."
|
||||
exit 1
|
||||
fi
|
||||
@@ -9,14 +9,14 @@ run_claude() {
|
||||
local allowed_tools="${3:-}"
|
||||
local output_file=$(mktemp)
|
||||
|
||||
# Build command
|
||||
local cmd="claude -p \"$prompt\""
|
||||
# Build command as an argv array so timeout wraps claude directly.
|
||||
local cmd=(claude -p "$prompt")
|
||||
if [ -n "$allowed_tools" ]; then
|
||||
cmd="$cmd --allowed-tools=$allowed_tools"
|
||||
cmd+=(--allowed-tools="$allowed_tools")
|
||||
fi
|
||||
|
||||
# Run Claude in headless mode with timeout
|
||||
if timeout "$timeout" bash -c "$cmd" > "$output_file" 2>&1; then
|
||||
if timeout "$timeout" "${cmd[@]}" > "$output_file" 2>&1; then
|
||||
cat "$output_file"
|
||||
rm -f "$output_file"
|
||||
return 0
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Integration Test: requesting-code-review skill
|
||||
# Verifies the code reviewer dispatched via the skill catches a planted bug
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
source "$SCRIPT_DIR/test-helpers.sh"
|
||||
|
||||
echo "========================================"
|
||||
echo " Integration Test: requesting-code-review"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "This test verifies the code reviewer subagent by:"
|
||||
echo " 1. Setting up a tiny project with a baseline commit"
|
||||
echo " 2. Adding a second commit that plants an obvious bug"
|
||||
echo " 3. Dispatching the code reviewer via the requesting-code-review skill"
|
||||
echo " 4. Verifying the reviewer flags the planted bug as Critical/Important"
|
||||
echo ""
|
||||
|
||||
TEST_PROJECT=$(create_test_project)
|
||||
echo "Test project: $TEST_PROJECT"
|
||||
trap "cleanup_test_project $TEST_PROJECT" EXIT
|
||||
|
||||
cd "$TEST_PROJECT"
|
||||
|
||||
# Baseline: a small "safe" implementation
|
||||
mkdir -p src
|
||||
cat > src/db.js <<'EOF'
|
||||
import { Database } from "./database-driver.js";
|
||||
|
||||
const db = new Database();
|
||||
|
||||
export async function findUserByEmail(email) {
|
||||
if (typeof email !== "string" || !email) {
|
||||
throw new Error("email required");
|
||||
}
|
||||
return db.query(
|
||||
"SELECT id, email, created_at FROM users WHERE email = ?",
|
||||
[email],
|
||||
);
|
||||
}
|
||||
EOF
|
||||
|
||||
cat > package.json <<'EOF'
|
||||
{ "name": "test-codereview", "version": "1.0.0", "type": "module" }
|
||||
EOF
|
||||
|
||||
git init --quiet
|
||||
git config user.email "test@test.com"
|
||||
git config user.name "Test User"
|
||||
git add .
|
||||
git commit -m "Initial: parameterized findUserByEmail" --quiet
|
||||
BASE_SHA=$(git rev-parse HEAD)
|
||||
|
||||
# Second commit: plant two real bugs
|
||||
# 1. SQL injection — switch from parameterized to string concatenation
|
||||
# 2. Logs the user's password hash on every successful login
|
||||
cat > src/db.js <<'EOF'
|
||||
import { Database } from "./database-driver.js";
|
||||
|
||||
const db = new Database();
|
||||
|
||||
export async function findUserByEmail(email) {
|
||||
return db.query(
|
||||
"SELECT id, email, password_hash, created_at FROM users WHERE email = '" + email + "'",
|
||||
);
|
||||
}
|
||||
|
||||
export async function login(email, password) {
|
||||
const user = await findUserByEmail(email);
|
||||
if (user && user.password_hash === hash(password)) {
|
||||
console.log("login success", { email, password_hash: user.password_hash });
|
||||
return user;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function hash(s) { return s; }
|
||||
EOF
|
||||
|
||||
git add .
|
||||
git commit -m "Refactor user lookup, add login" --quiet
|
||||
HEAD_SHA=$(git rev-parse HEAD)
|
||||
|
||||
echo ""
|
||||
echo "Planted bugs in $BASE_SHA..$HEAD_SHA:"
|
||||
echo " - SQL injection (string concat instead of parameterized query)"
|
||||
echo " - Password hash logged in plaintext on every successful login"
|
||||
echo " - hash() is the identity function (passwords stored & compared in plaintext)"
|
||||
echo ""
|
||||
|
||||
OUTPUT_FILE="$TEST_PROJECT/claude-output.txt"
|
||||
|
||||
PROMPT="I just finished a refactor. The change is between commits $BASE_SHA and $HEAD_SHA on the current branch.
|
||||
|
||||
Use the superpowers:requesting-code-review skill to review these changes before I merge. Follow the skill exactly: dispatch the code reviewer subagent with the template, give the subagent the SHA range, and report back what it found.
|
||||
|
||||
Print the reviewer's full output."
|
||||
|
||||
# Run claude from inside the test project so its session JSONL lands in a
|
||||
# project-specific directory under ~/.claude/projects/, isolated from any
|
||||
# other concurrent claude sessions.
|
||||
echo "Running Claude (plugin-dir: $PLUGIN_DIR, cwd: $TEST_PROJECT)..."
|
||||
echo "================================================================================"
|
||||
cd "$TEST_PROJECT" && timeout 600 claude -p "$PROMPT" \
|
||||
--plugin-dir "$PLUGIN_DIR" \
|
||||
--permission-mode bypassPermissions 2>&1 | tee "$OUTPUT_FILE" || {
|
||||
echo ""
|
||||
echo "================================================================================"
|
||||
echo "EXECUTION FAILED (exit code: $?)"
|
||||
exit 1
|
||||
}
|
||||
echo "================================================================================"
|
||||
|
||||
echo ""
|
||||
echo "Analyzing reviewer output..."
|
||||
echo ""
|
||||
|
||||
# Find the session transcript. Because we ran claude from $TEST_PROJECT (a
|
||||
# unique tmp dir), its sessions live in their own ~/.claude/projects/ folder.
|
||||
# Resolve the real path (macOS mktemp returns /var/... but claude normalizes
|
||||
# it to /private/var/...) and replicate claude's normalization (every
|
||||
# non-alphanumeric char becomes `-`).
|
||||
TEST_PROJECT_REAL=$(cd "$TEST_PROJECT" && pwd -P)
|
||||
SESSION_DIR="$HOME/.claude/projects/$(echo "$TEST_PROJECT_REAL" | sed 's|[^a-zA-Z0-9]|-|g')"
|
||||
# `|| true` prevents pipefail killing the script if ls gets SIGPIPE'd by head.
|
||||
SESSION_FILE=$(ls -t "$SESSION_DIR"/*.jsonl 2>/dev/null | head -1 || true)
|
||||
|
||||
FAILED=0
|
||||
|
||||
echo "=== Verification Tests ==="
|
||||
echo ""
|
||||
|
||||
# Test 1: Skill was actually invoked, and a subagent was actually dispatched
|
||||
echo "Test 1: requesting-code-review skill invoked + reviewer subagent dispatched..."
|
||||
if [ -z "$SESSION_FILE" ] || [ ! -f "$SESSION_FILE" ]; then
|
||||
echo " [FAIL] Could not locate session transcript in $SESSION_DIR"
|
||||
FAILED=$((FAILED + 1))
|
||||
elif ! grep -q '"skill":"superpowers:requesting-code-review"' "$SESSION_FILE"; then
|
||||
echo " [FAIL] requesting-code-review skill was not invoked"
|
||||
echo " Session: $SESSION_FILE"
|
||||
FAILED=$((FAILED + 1))
|
||||
elif ! grep -q '"name":"Agent"' "$SESSION_FILE"; then
|
||||
echo " [FAIL] Skill ran but no subagent was dispatched"
|
||||
FAILED=$((FAILED + 1))
|
||||
else
|
||||
echo " [PASS] Skill invoked and subagent dispatched"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 2: Reviewer caught the SQL injection
|
||||
echo "Test 2: SQL injection flagged..."
|
||||
if grep -qiE "sql injection|injection|string concat|parameterize|prepared statement|sanitiz" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Reviewer flagged the SQL injection vector"
|
||||
else
|
||||
echo " [FAIL] Reviewer missed the SQL injection — most obvious planted bug"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 3: Reviewer caught the credential / password issue (either logging or no real hashing)
|
||||
echo "Test 3: Credential handling issue flagged..."
|
||||
if grep -qiE "password|credential|secret|plaintext|log.*hash|hash.*log|sensitive" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Reviewer flagged a credential / password handling issue"
|
||||
else
|
||||
echo " [FAIL] Reviewer missed the password/credential issues"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 4: Reviewer marked at least one issue as Critical or Important (not just Minor)
|
||||
echo "Test 4: Severity classification..."
|
||||
if grep -qiE "critical|important|severe|high.*risk|security" "$OUTPUT_FILE"; then
|
||||
echo " [PASS] Reviewer classified findings at Critical/Important severity"
|
||||
else
|
||||
echo " [FAIL] Reviewer did not classify findings as Critical or Important"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 5: Reviewer did NOT approve the diff for merge
|
||||
echo "Test 5: Reviewer verdict..."
|
||||
# A correct reviewer says No or "With fixes". A broken/sycophantic reviewer says Yes/Ready.
|
||||
if grep -qiE "ready to merge.*yes|approved.*for merge|^\s*yes\s*$|safe to merge" "$OUTPUT_FILE" \
|
||||
&& ! grep -qiE "ready to merge.*no|with fixes|do not merge|not ready|block.*merge" "$OUTPUT_FILE"; then
|
||||
echo " [FAIL] Reviewer approved a diff with planted Critical bugs"
|
||||
FAILED=$((FAILED + 1))
|
||||
else
|
||||
echo " [PASS] Reviewer did not approve the diff"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "========================================"
|
||||
echo " Test Summary"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo "STATUS: PASSED"
|
||||
echo "The code reviewer correctly:"
|
||||
echo " ✓ Was dispatched via the requesting-code-review skill"
|
||||
echo " ✓ Flagged the SQL injection"
|
||||
echo " ✓ Flagged the credential handling issues"
|
||||
echo " ✓ Classified findings at Critical/Important severity"
|
||||
echo " ✓ Did not approve the diff for merge"
|
||||
exit 0
|
||||
else
|
||||
echo "STATUS: FAILED"
|
||||
echo "Failed $FAILED verification tests"
|
||||
echo ""
|
||||
echo "Output saved to: $OUTPUT_FILE"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,6 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
# Integration Test: subagent-driven-development workflow
|
||||
# Actually executes a plan and verifies the new workflow behaviors
|
||||
#
|
||||
# Drill coverage: evals/scenarios/sdd-rejects-extra-features.yaml covers the
|
||||
# YAGNI enforcement subset (forbidden exports + reviewer-as-gate semantics)
|
||||
# and is stricter on that axis. This bash test additionally asserts:
|
||||
# - >=3 git commits (initial + per-task commits, exercising SDD's
|
||||
# commit-per-task workflow shape)
|
||||
# - >=2 Claude Code subagent dispatches via Agent or Task (drill only asserts >=1)
|
||||
# - Claude Code task-tracking tool usage (drill makes no assertion)
|
||||
# - test/math.test.js exists (drill relies on `npm test` succeeding)
|
||||
# - analyze-token-usage.py token-budget telemetry
|
||||
# Kept until those assertions are added to drill or explicitly retired.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
@@ -213,13 +224,13 @@ else
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Test 3: TodoWrite was used for tracking
|
||||
# Test 3: Claude Code task-tracking tool was used
|
||||
echo "Test 3: Task tracking..."
|
||||
todo_count=$(grep -c '"name":"TodoWrite"' "$SESSION_FILE" || echo "0")
|
||||
todo_count=$(grep -cE '"name":"(TodoWrite|TaskCreate|TaskUpdate|TaskList|TaskGet)"' "$SESSION_FILE" || echo "0")
|
||||
if [ "$todo_count" -ge 1 ]; then
|
||||
echo " [PASS] TodoWrite used $todo_count time(s) for task tracking"
|
||||
echo " [PASS] Task tracking used $todo_count time(s)"
|
||||
else
|
||||
echo " [FAIL] TodoWrite not used"
|
||||
echo " [FAIL] No Claude Code task-tracking tool used"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
@@ -1,18 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test: subagent-driven-development skill
|
||||
# Verifies that the skill is loaded and follows correct workflow
|
||||
#
|
||||
# No drill coverage: this test asks the agent to *describe* SDD (string-
|
||||
# matches its verbal explanation against expected keywords like
|
||||
# "self-review", "skeptical", "worktree", "Step 1", "loop"). Drill scenarios
|
||||
# test behavior (real subagent dispatch, plan-following, review loops),
|
||||
# not description-recall. Kept by design.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/test-helpers.sh"
|
||||
|
||||
CLAUDE_PROMPT_TIMEOUT="${CLAUDE_PROMPT_TIMEOUT:-90}"
|
||||
|
||||
echo "=== Test: subagent-driven-development skill ==="
|
||||
echo ""
|
||||
|
||||
# Test 1: Verify skill can be loaded
|
||||
echo "Test 1: Skill loading..."
|
||||
|
||||
output=$(run_claude "What is the subagent-driven-development skill? Describe its key steps briefly." 30)
|
||||
output=$(run_claude "What is the subagent-driven-development skill? Describe its key steps briefly." "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "subagent-driven-development\|Subagent-Driven Development\|Subagent Driven" "Skill is recognized"; then
|
||||
: # pass
|
||||
@@ -31,9 +39,11 @@ echo ""
|
||||
# Test 2: Verify skill describes correct workflow order
|
||||
echo "Test 2: Workflow ordering..."
|
||||
|
||||
output=$(run_claude "In the subagent-driven-development skill, what comes first: spec compliance review or code quality review? Be specific about the order." 30)
|
||||
output=$(run_claude "In the subagent-driven-development skill, what comes first: spec compliance review or code quality review? Answer using exactly this structure:
|
||||
First: <review type>
|
||||
Second: <review type>" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_order "$output" "spec.*compliance" "code.*quality" "Spec compliance before code quality"; then
|
||||
if assert_order "$output" "First:.*spec.*compliance" "Second:.*code.*quality" "Spec compliance before code quality"; then
|
||||
: # pass
|
||||
else
|
||||
exit 1
|
||||
@@ -44,15 +54,17 @@ echo ""
|
||||
# Test 3: Verify self-review is mentioned
|
||||
echo "Test 3: Self-review requirement..."
|
||||
|
||||
output=$(run_claude "Does the subagent-driven-development skill require implementers to do self-review? What should they check?" 30)
|
||||
output=$(run_claude "Does the subagent-driven-development skill require implementers to self-review before handoff, and can self-review replace the external reviews? Answer using exactly this structure:
|
||||
Self-review required: <yes or no>
|
||||
Self-review replaces external review: <yes or no>" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "self-review\|self review" "Mentions self-review"; then
|
||||
if assert_contains "$output" "Self-review required:.*yes" "Mentions self-review"; then
|
||||
: # pass
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if assert_contains "$output" "completeness\|Completeness" "Checks completeness"; then
|
||||
if assert_contains "$output" "Self-review replaces external review:.*no" "Self-review does not replace external review"; then
|
||||
: # pass
|
||||
else
|
||||
exit 1
|
||||
@@ -63,7 +75,7 @@ echo ""
|
||||
# Test 4: Verify plan is read once
|
||||
echo "Test 4: Plan reading efficiency..."
|
||||
|
||||
output=$(run_claude "In subagent-driven-development, how many times should the controller read the plan file? When does this happen?" 30)
|
||||
output=$(run_claude "In subagent-driven-development, how many times should the controller read the plan file? When does this happen?" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "once\|one time\|single" "Read plan once"; then
|
||||
: # pass
|
||||
@@ -82,7 +94,7 @@ echo ""
|
||||
# Test 5: Verify spec compliance reviewer is skeptical
|
||||
echo "Test 5: Spec compliance reviewer mindset..."
|
||||
|
||||
output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" 30)
|
||||
output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
|
||||
: # pass
|
||||
@@ -101,7 +113,7 @@ echo ""
|
||||
# Test 6: Verify review loops
|
||||
echo "Test 6: Review loop requirements..."
|
||||
|
||||
output=$(run_claude "In subagent-driven-development, what happens if a reviewer finds issues? Is it a one-time review or a loop?" 30)
|
||||
output=$(run_claude "In subagent-driven-development, what happens if a reviewer finds issues? Is it a one-time review or a loop?" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "loop\|again\|repeat\|until.*approved\|until.*compliant" "Review loops mentioned"; then
|
||||
: # pass
|
||||
@@ -120,7 +132,9 @@ echo ""
|
||||
# Test 7: Verify full task text is provided
|
||||
echo "Test 7: Task context provision..."
|
||||
|
||||
output=$(run_claude "In subagent-driven-development, how does the controller provide task information to the implementer subagent? Does it make them read a file or provide it directly?" 30)
|
||||
output=$(run_claude "In subagent-driven-development, how does the controller provide task information to the implementer subagent? Answer using exactly this structure:
|
||||
Controller provides: <directly or by file>
|
||||
Implementer must read plan file: <yes or no>" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "provide.*directly\|full.*text\|paste\|include.*prompt" "Provides text directly"; then
|
||||
: # pass
|
||||
@@ -128,7 +142,7 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if assert_not_contains "$output" "read.*file\|open.*file" "Doesn't make subagent read file"; then
|
||||
if assert_contains "$output" "Implementer must read plan file:.*no" "Doesn't make subagent read file"; then
|
||||
: # pass
|
||||
else
|
||||
exit 1
|
||||
@@ -139,7 +153,7 @@ echo ""
|
||||
# Test 8: Verify worktree requirement
|
||||
echo "Test 8: Worktree requirement..."
|
||||
|
||||
output=$(run_claude "What workflow skills are required before using subagent-driven-development? List any prerequisites or required skills." 30)
|
||||
output=$(run_claude "What workflow skills are required before using subagent-driven-development? List any prerequisites or required skills." "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "using-git-worktrees\|worktree" "Mentions worktree requirement"; then
|
||||
: # pass
|
||||
@@ -152,7 +166,7 @@ echo ""
|
||||
# Test 9: Verify main branch warning
|
||||
echo "Test 9: Main branch red flag..."
|
||||
|
||||
output=$(run_claude "In subagent-driven-development, is it okay to start implementation directly on the main branch?" 30)
|
||||
output=$(run_claude "In subagent-driven-development, is it okay to start implementation directly on the main branch?" "$CLAUDE_PROMPT_TIMEOUT")
|
||||
|
||||
if assert_contains "$output" "worktree\|feature.*branch\|not.*main\|never.*main\|avoid.*main\|don't.*main\|consent\|permission" "Warns against main branch"; then
|
||||
: # pass
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
# Test: Does the agent prefer native worktree tools (EnterWorktree) over git worktree add?
|
||||
# Framework: RED-GREEN-REFACTOR per testing-skills-with-subagents.md
|
||||
#
|
||||
# Drill coverage: evals/scenarios/worktree-creation-under-pressure.yaml lifts
|
||||
# only the PRESSURE phase (existing .worktrees/ + urgency framing). The RED
|
||||
# and GREEN baselines below are not covered by drill — kept here so the
|
||||
# RED-GREEN-REFACTOR validation remains rerunnable end-to-end.
|
||||
#
|
||||
# RED: Skill without Step 1a (no native tool preference). Agent should use git worktree add.
|
||||
# GREEN: Skill with Step 1a (explicit tool naming + consent bridge). Agent should use EnterWorktree.
|
||||
# PRESSURE: Same as GREEN but under time pressure with existing .worktrees/ dir.
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regression check: Superpowers should not route new worktrees through the old
|
||||
# global worktree directory.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
USING_SKILL="$REPO_ROOT/skills/using-git-worktrees/SKILL.md"
|
||||
FINISHING_SKILL="$REPO_ROOT/skills/finishing-a-development-branch/SKILL.md"
|
||||
ROTOTILL_SPEC="$REPO_ROOT/docs/superpowers/specs/2026-04-06-worktree-rototill-design.md"
|
||||
ROTOTILL_PLAN="$REPO_ROOT/docs/superpowers/plans/2026-04-06-worktree-rototill.md"
|
||||
|
||||
failures=0
|
||||
|
||||
assert_contains() {
|
||||
local file="$1"
|
||||
local pattern="$2"
|
||||
local label="$3"
|
||||
|
||||
if grep -Fq "$pattern" "$file"; then
|
||||
echo " [PASS] $label"
|
||||
else
|
||||
echo " [FAIL] $label"
|
||||
echo " Expected to find: $pattern"
|
||||
echo " In file: $file"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_not_contains() {
|
||||
local file="$1"
|
||||
local pattern="$2"
|
||||
local label="$3"
|
||||
|
||||
if grep -Fq "$pattern" "$file"; then
|
||||
echo " [FAIL] $label"
|
||||
echo " Did not expect to find: $pattern"
|
||||
echo " In file: $file"
|
||||
failures=$((failures + 1))
|
||||
else
|
||||
echo " [PASS] $label"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== Worktree Path Policy Test ==="
|
||||
echo ""
|
||||
|
||||
assert_not_contains "$USING_SKILL" "~/.config/superpowers/worktrees" "using-git-worktrees does not mention old global path"
|
||||
assert_not_contains "$USING_SKILL" "global legacy" "using-git-worktrees does not use unclear global legacy shorthand"
|
||||
assert_not_contains "$USING_SKILL" "Global path" "using-git-worktrees has no global path quick-reference row"
|
||||
assert_contains "$USING_SKILL" 'default to `.worktrees/` at the project root' "using-git-worktrees defaults new manual worktrees to .worktrees/"
|
||||
|
||||
assert_not_contains "$FINISHING_SKILL" "~/.config/superpowers/worktrees" "finishing-a-development-branch does not treat old global path as owned"
|
||||
assert_contains "$FINISHING_SKILL" '`.worktrees/` or `worktrees/`' "finishing-a-development-branch keeps project-local cleanup ownership"
|
||||
|
||||
assert_not_contains "$ROTOTILL_SPEC" "~/.config/superpowers/worktrees" "rototill spec does not preserve old global path policy"
|
||||
assert_not_contains "$ROTOTILL_PLAN" "~/.config/superpowers/worktrees" "rototill plan does not preserve old global path policy"
|
||||
assert_not_contains "$ROTOTILL_PLAN" "legacy path compat" "rototill plan does not advertise legacy path compatibility"
|
||||
|
||||
echo ""
|
||||
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
echo "STATUS: FAILED ($failures failures)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "STATUS: PASSED"
|
||||
Reference in New Issue
Block a user