actions/claude-code-action
1
0
Fork 0
mirror of https://github.com/anthropics/claude-code-action.git synced 2026-08-22 03:18:54 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
a2cac87e2759e8936c7e05edb8001b9298a2b5cb
claude-code-action/.github/workflows/test-structured-output.yml
T
Ashwin BhatandGitHub a2cac87e27 ci: skip Claude-backed test jobs on fork PRs (#1655) 
* ci: skip Claude-backed test jobs on fork PRs

Jobs that run the action against the Claude API authenticate via workload
identity federation, which fork PRs cannot mint an OIDC token for, so they
always failed on external contributions. Gate each such job on the PR head
repo matching the base repo; push and workflow_dispatch runs are unaffected.

No-Verification-Needed: CI workflow config only, exercised by Actions on the PR

* test: pin the bare remote's initial branch in fetch-depth test

The shallow-clone case created its bare remote with a plain git init, so
HEAD pointed at whatever init.defaultBranch resolves to (master on CI)
while the test only pushed main. git clone --depth=1 implies
--single-branch, and with a dangling remote HEAD it produces an empty,
non-shallow clone, so the is-shallow assertion failed on runners whose
default branch is not main.

No-Verification-Needed: test-only change
2026-08-14 16:40:34 -07:00

330 lines
13 KiB
YAML
Raw Blame History

name: Test Structured Outputs
on:
pull_request:
workflow_dispatch:
workflow_call:
# The Claude API is authenticated via workload identity federation: id-token
# lets the action mint the GitHub OIDC token it exchanges for a short-lived
# access token. See docs/setup.md.
permissions:
contents: read
id-token: write
jobs:
test-basic-types:
name: Test Basic Type Conversions
# Skip on fork PRs since they can't mint the OIDC token used for Claude API auth
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Test with explicit values
id: test
uses: ./base-action
with:
prompt: |
Run this command: echo "test"
Then return EXACTLY these values:
- text_field: "hello"
- number_field: 42
- boolean_true: true
- boolean_false: false
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"text_field":{"type":"string"},"number_field":{"type":"number"},"boolean_true":{"type":"boolean"},"boolean_false":{"type":"boolean"}},"required":["text_field","number_field","boolean_true","boolean_false"]}'
- name: Verify outputs
run: |
# Parse the structured_output JSON
OUTPUT='${{ steps.test.outputs.structured_output }}'
# Test string pass-through
TEXT_FIELD=$(echo "$OUTPUT" | jq -r '.text_field')
if [ "$TEXT_FIELD" != "hello" ]; then
echo "❌ String: expected 'hello', got '$TEXT_FIELD'"
exit 1
fi
# Test number → string conversion
NUMBER_FIELD=$(echo "$OUTPUT" | jq -r '.number_field')
if [ "$NUMBER_FIELD" != "42" ]; then
echo "❌ Number: expected '42', got '$NUMBER_FIELD'"
exit 1
fi
# Test boolean → "true" conversion
BOOLEAN_TRUE=$(echo "$OUTPUT" | jq -r '.boolean_true')
if [ "$BOOLEAN_TRUE" != "true" ]; then
echo "❌ Boolean true: expected 'true', got '$BOOLEAN_TRUE'"
exit 1
fi
# Test boolean → "false" conversion
BOOLEAN_FALSE=$(echo "$OUTPUT" | jq -r '.boolean_false')
if [ "$BOOLEAN_FALSE" != "false" ]; then
echo "❌ Boolean false: expected 'false', got '$BOOLEAN_FALSE'"
exit 1
fi
echo "✅ All basic type conversions correct"
test-complex-types:
name: Test Arrays and Objects
# Skip on fork PRs since they can't mint the OIDC token used for Claude API auth
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Test complex types
id: test
uses: ./base-action
with:
prompt: |
Run: echo "ready"
Return EXACTLY:
- items: ["apple", "banana", "cherry"]
- config: {"key": "value", "count": 3}
- empty_array: []
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"items":{"type":"array","items":{"type":"string"}},"config":{"type":"object"},"empty_array":{"type":"array"}},"required":["items","config","empty_array"]}'
- name: Verify JSON stringification
run: |
# Parse the structured_output JSON
OUTPUT='${{ steps.test.outputs.structured_output }}'
# Arrays should be JSON stringified
if ! echo "$OUTPUT" | jq -e '.items | length == 3' > /dev/null; then
echo "❌ Array not properly formatted"
echo "$OUTPUT" | jq '.items'
exit 1
fi
# Objects should be JSON stringified
if ! echo "$OUTPUT" | jq -e '.config.key == "value"' > /dev/null; then
echo "❌ Object not properly formatted"
echo "$OUTPUT" | jq '.config'
exit 1
fi
# Empty arrays should work
if ! echo "$OUTPUT" | jq -e '.empty_array | length == 0' > /dev/null; then
echo "❌ Empty array not properly formatted"
echo "$OUTPUT" | jq '.empty_array'
exit 1
fi
echo "✅ All complex types handled correctly"
test-edge-cases:
name: Test Edge Cases
# Skip on fork PRs since they can't mint the OIDC token used for Claude API auth
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Test edge cases
id: test
uses: ./base-action
with:
prompt: |
Run: echo "test"
Return EXACTLY:
- zero: 0
- empty_string: ""
- negative: -5
- decimal: 3.14
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"zero":{"type":"number"},"empty_string":{"type":"string"},"negative":{"type":"number"},"decimal":{"type":"number"}},"required":["zero","empty_string","negative","decimal"]}'
- name: Verify edge cases
run: |
# Parse the structured_output JSON
OUTPUT='${{ steps.test.outputs.structured_output }}'
# Zero should be "0", not empty or falsy
ZERO=$(echo "$OUTPUT" | jq -r '.zero')
if [ "$ZERO" != "0" ]; then
echo "❌ Zero: expected '0', got '$ZERO'"
exit 1
fi
# Empty string should be empty (not "null" or missing)
EMPTY_STRING=$(echo "$OUTPUT" | jq -r '.empty_string')
if [ "$EMPTY_STRING" != "" ]; then
echo "❌ Empty string: expected '', got '$EMPTY_STRING'"
exit 1
fi
# Negative numbers should work
NEGATIVE=$(echo "$OUTPUT" | jq -r '.negative')
if [ "$NEGATIVE" != "-5" ]; then
echo "❌ Negative: expected '-5', got '$NEGATIVE'"
exit 1
fi
# Decimals should preserve precision
DECIMAL=$(echo "$OUTPUT" | jq -r '.decimal')
if [ "$DECIMAL" != "3.14" ]; then
echo "❌ Decimal: expected '3.14', got '$DECIMAL'"
exit 1
fi
echo "✅ All edge cases handled correctly"
test-name-sanitization:
name: Test Output Name Sanitization
# Skip on fork PRs since they can't mint the OIDC token used for Claude API auth
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Test special characters in field names
id: test
uses: ./base-action
with:
prompt: |
Run: echo "test"
Return EXACTLY: {test-result: "passed", item_count: 10}
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"test-result":{"type":"string"},"item_count":{"type":"number"}},"required":["test-result","item_count"]}'
- name: Verify sanitized names work
run: |
# Parse the structured_output JSON
OUTPUT='${{ steps.test.outputs.structured_output }}'
# Hyphens should be preserved in the JSON
TEST_RESULT=$(echo "$OUTPUT" | jq -r '.["test-result"]')
if [ "$TEST_RESULT" != "passed" ]; then
echo "❌ Hyphenated name failed: expected 'passed', got '$TEST_RESULT'"
exit 1
fi
# Underscores should work
ITEM_COUNT=$(echo "$OUTPUT" | jq -r '.item_count')
if [ "$ITEM_COUNT" != "10" ]; then
echo "❌ Underscore name failed: expected '10', got '$ITEM_COUNT'"
exit 1
fi
echo "✅ Name sanitization works"
test-execution-file-structure:
name: Test Execution File Format
# Skip on fork PRs since they can't mint the OIDC token used for Claude API auth
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Run with structured output
id: test
uses: ./base-action
with:
prompt: "Run: echo 'complete'. Return: {done: true}"
anthropic_federation_rule_id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }}
anthropic_organization_id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }}
anthropic_service_account_id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }}
claude_args: |
--allowedTools Bash
--json-schema '{"type":"object","properties":{"done":{"type":"boolean"}},"required":["done"]}'
- name: Verify execution file contains structured_output
run: |
FILE="${{ steps.test.outputs.execution_file }}"
# Check file exists
if [ ! -f "$FILE" ]; then
echo "❌ Execution file missing"
exit 1
fi
# Check for structured_output field
if ! jq -e '.[] | select(.type == "result") | .structured_output' "$FILE" > /dev/null; then
echo "❌ No structured_output in execution file"
cat "$FILE"
exit 1
fi
# Verify the actual value
DONE=$(jq -r '.[] | select(.type == "result") | .structured_output.done' "$FILE")
if [ "$DONE" != "true" ]; then
echo "❌ Wrong value in execution file"
exit 1
fi
echo "✅ Execution file format correct"
test-summary:
name: Summary
runs-on: ubuntu-latest
needs:
- test-basic-types
- test-complex-types
- test-edge-cases
- test-name-sanitization
- test-execution-file-structure
if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
steps:
- name: Generate Summary
run: |
echo "# Structured Output Tests (Optimized)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Fast, deterministic tests using explicit prompts" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Test | Result |" >> $GITHUB_STEP_SUMMARY
echo "|------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Basic Types | ${{ needs.test-basic-types.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY
echo "| Complex Types | ${{ needs.test-complex-types.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY
echo "| Edge Cases | ${{ needs.test-edge-cases.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY
echo "| Name Sanitization | ${{ needs.test-name-sanitization.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY
echo "| Execution File | ${{ needs.test-execution-file-structure.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY
# Check if all passed
ALL_PASSED=${{
needs.test-basic-types.result == 'success' &&
needs.test-complex-types.result == 'success' &&
needs.test-edge-cases.result == 'success' &&
needs.test-name-sanitization.result == 'success' &&
needs.test-execution-file-structure.result == 'success'
}}
if [ "$ALL_PASSED" = "true" ]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "## ✅ All Tests Passed" >> $GITHUB_STEP_SUMMARY
else
echo "" >> $GITHUB_STEP_SUMMARY
echo "## ❌ Some Tests Failed" >> $GITHUB_STEP_SUMMARY
exit 1
fi
Reference in New Issue View Git Blame Copy Permalink