139 lines
4.4 KiB
Bash
139 lines
4.4 KiB
Bash
#!/bin/bash
|
||
# Batch-test prompts against the test-prompts executable
|
||
|
||
SUCCESS_COUNT=0
|
||
FAIL_COUNT=0
|
||
TOTAL_COUNT=0
|
||
|
||
TEST_PROMPTS_EXEC="./bin/test-prompts"
|
||
|
||
if [ ! -f "$TEST_PROMPTS_EXEC" ]; then
|
||
echo "✗ ERROR: '$TEST_PROMPTS_EXEC' not found. Please build it first."
|
||
exit 1
|
||
fi
|
||
|
||
echo "=== Batch Prompts Test ==="
|
||
echo
|
||
|
||
# --- Test Functions ---
|
||
|
||
# Test for inputs that should successfully parse and generate a prompt
|
||
test_positive() {
|
||
local input="$1"
|
||
local description="$2"
|
||
TOTAL_COUNT=$((TOTAL_COUNT+1))
|
||
|
||
echo "---"
|
||
echo "Testing (should succeed): $description"
|
||
# echo "Input: $input"
|
||
|
||
# Run the command, capture output
|
||
output=$("$TEST_PROMPTS_EXEC" "$input")
|
||
exit_code=$?
|
||
|
||
# Check for command execution error
|
||
if [ $exit_code -ne 0 ]; then
|
||
echo " ✗ FAIL: test-prompts command failed with exit code $exit_code."
|
||
FAIL_COUNT=$((FAIL_COUNT+1))
|
||
echo "---"
|
||
return
|
||
fi
|
||
|
||
# Check for a valid question and category in the EXTRACTION section
|
||
extraction_json=$(echo "$output" | awk '/^=== EXTRACTION ===$/,/^=== SUMMARY ===$/' | sed '1d;$d')
|
||
question=$(echo "$extraction_json" | jq -r '.question // ""')
|
||
category=$(echo "$extraction_json" | jq -r '.category // ""')
|
||
|
||
if [ -n "$question" ] && [ "$question" != "null" ] && [ "$question" != "" ] && [ -n "$category" ]; then
|
||
echo " ✓ SUCCESS: New prompt created."
|
||
echo " Question: $question"
|
||
echo " Category: $category"
|
||
SUCCESS_COUNT=$((SUCCESS_COUNT+1))
|
||
else
|
||
echo " ✗ FAIL: Did not generate a valid question or category."
|
||
# echo "$output"
|
||
FAIL_COUNT=$((FAIL_COUNT+1))
|
||
fi
|
||
# echo
|
||
}
|
||
|
||
# Test for inputs that should be ignored (no prompt/schedule generated)
|
||
test_negative() {
|
||
local input="$1"
|
||
local expected_error="$2"
|
||
TOTAL_COUNT=$((TOTAL_COUNT+1))
|
||
|
||
echo "---"
|
||
echo "Testing (should be ignored): $input"
|
||
|
||
output=$("$TEST_PROMPTS_EXEC" "$input" 2>&1)
|
||
exit_code=$?
|
||
|
||
if [ $exit_code -ne 0 ]; then
|
||
echo " ✗ FAIL: test-prompts command failed unexpectedly with exit code $exit_code."
|
||
FAIL_COUNT=$((FAIL_COUNT+1))
|
||
echo "---"
|
||
return
|
||
fi
|
||
|
||
# Extract the Triage Error from the output
|
||
triage_error=$(echo "$output" | grep -A 4 "=== TRIAGE ===" | grep "Error:" | awk -F': ' '{print $2}')
|
||
|
||
if [[ -n "$triage_error" ]] && [[ "$triage_error" == *"$expected_error"* ]]; then
|
||
echo " ✓ SUCCESS: Correctly triaged with error '$triage_error'"
|
||
SUCCESS_COUNT=$((SUCCESS_COUNT+1))
|
||
else
|
||
echo " ✗ FAIL: Did not produce the expected triage error. Expected '$expected_error', got '$triage_error'."
|
||
echo "$output"
|
||
FAIL_COUNT=$((FAIL_COUNT+1))
|
||
fi
|
||
}
|
||
|
||
# --- Test Data ---
|
||
|
||
declare -a positive_prompts=(
|
||
"I went to the gym and did 20 push-ups, 30 pull-ups, and ran 5 miles on the treadmill."
|
||
"Today for lunch I had a chicken salad and a glass of water, and for dinner I had a steak with a side of potatoes."
|
||
"I took my morning pills: 10mg of Lisinopril and a multivitamin."
|
||
"I have a follow-up with my cardiologist on Monday and a dental cleaning on Wednesday."
|
||
"Ich habe heute Morgen 30 Minuten Yoga gemacht."
|
||
"Ik heb een afspraak met de fysiotherapeut op dinsdag."
|
||
"J'ai une douleur au genou depuis 3 jours."
|
||
"Я измерил уровень сахара в крови, он 5.8."
|
||
"My period started today."
|
||
"I feel really tired and fatigued."
|
||
"I have a new prescription for Warfarin."
|
||
"I need to stop eating sugar."
|
||
"my blood pressure is 87 over 123"
|
||
"I went to the dentist; next Monday I need to do a root canal"
|
||
"I'm pregnant!"
|
||
)
|
||
|
||
declare -A negative_prompts
|
||
negative_prompts["What is the capital of Mexico?"]="not_health_related"
|
||
negative_prompts["I have a headache, what should I do?"]="no_medical_advice"
|
||
negative_prompts["Set a timer for 5 minutes."]="not_health_related"
|
||
negative_prompts["What's the weather like tomorrow?"]="not_health_related"
|
||
|
||
# --- Test Execution ---
|
||
|
||
for prompt in "${positive_prompts[@]}"; do
|
||
test_positive "$prompt" "$prompt"
|
||
done
|
||
|
||
for prompt in "${!negative_prompts[@]}"; do
|
||
test_negative "$prompt" "${negative_prompts[$prompt]}"
|
||
done
|
||
|
||
# --- Summary ---
|
||
echo
|
||
echo "---"
|
||
echo "=== Test Summary ==="
|
||
echo "Total Tests: $TOTAL_COUNT"
|
||
echo "✓ Successes: $SUCCESS_COUNT"
|
||
echo "✗ Failures: $FAIL_COUNT"
|
||
echo "---"
|
||
|
||
if [ "$FAIL_COUNT" -gt 0 ]; then
|
||
exit 1
|
||
fi |