check_coverage.py

import sys
sys.path.insert(0, 'projects/telco-troubleshooting/src')

import json
from telco_agent.track_b_agent import TrackBAgent
from telco_agent.track_b_client import TrackBClient

# Load test data
with open('data/Phase_1/track_b/test.json', 'r', encoding='utf-8') as f:
    test_data = json.load(f)

agent = TrackBAgent(client=TrackBClient())

answered = 0
total = len(test_data)
unanswered = []

print("Checking Track B coverage...")
print(f"Total questions: {total}")
print()

for item in test_data:
    qid = int(item['task']['id'])
    question = item['task']['question']
    
    try:
        answer = agent.solve(question_number=qid, question=question)
        if answer and answer.strip():
            answered += 1
            print(f"Q{qid:2d}: [OK] Answered ({len(answer)} chars)")
        else:
            unanswered.append(qid)
            print(f"Q{qid:2d}: [EMPTY]")
    except Exception as e:
        unanswered.append(qid)
        print(f"Q{qid:2d}: [ERROR] {str(e)[:50]}")

print()
print(f"Coverage: {answered}/{total} = {100*answered/total:.1f}%")
print(f"Unanswered: {unanswered}")