Examples for Jarvis_tf Test Suite

---
.llm_settings: &llm_settings
  model: "anthropic.claude-3-haiku-20240307-v1:0"
  temperature: 0
  max_tokens: 1000
#end .llm_settings

metadata:
  name: Fleming Rosters Test Suite (Simple)
  description: Simple roster queries for Fleming Hospital.
  organization_key: fleming
#end metadata
endpoint:
  type: v2
  query_engine: llm
#end endpoint

test_cases:
  - name: cardio query
    query:
      query_text: cardio on call
      profile: # Jarvis v2
        tenant: "hospital:fleming"
      #end profile
    #end query

    asserts:
      - type: regexes
        regexes: ["[Cc]ardiology"]
        substrings: ["after oh", daily]
        case_sensitive: false
        minimum_should_match: 2 # refers regex not the variables
      - type: inverse_regexes
        regexes: ["${department} \\(After OH\\)"]
        case_sensitive: false
        minimum_should_match: 1 # refers regex not the variables
      - type: regexes_count
        regexes: ["Cardiology"]
        substrings: ["Cardiology"]
        case_sensitive: false
        equals: 2
      - type: regexes_count
        regexes: ["After OH"]
        substrings: ["Cardiology"]
        case_sensitive: false
        maximum: 2
      - type: message_type
        message_types: ["text"]
    #end asserts
  #end test_case1
  - name: ed query
    query:
      query_text: cardio on call
      profile: # Jarvis v1
        uid: jarvisdemoqueryflow123456
        organization_key: ktph
        first_name: John
        last_name: Smith
        department: Cardiology
        designation: Registrar
        tags: [AHFS_accessible, "botmd:internal", "botmd:jarvis", calculators_accessible, Uptodate_accessible, Mediview_accessible]
        acl_dnf: ["hospital:hospital=fleming"]
      #end profile
      #end query

    asserts:
      # contextual_precision, contextual_recall, contextual_relevancy, correctness, faithfulness, hallucination
      - type: answer_relevancy
        llm_settings: *llm_settings
        minimum: 0.5
      - type: contextual_precision
        llm_settings: *llm_settings
        minimum: 0.5
        expected_output: Hello world
      - type: contextual_recall
        llm_settings: *llm_settings
        minimum: 0.5
        expected_output: Hello world
      - type: contextual_relevancy
        llm_settings: *llm_settings
        minimum: 0.5
      - type: correctness
        llm_settings: *llm_settings
        minimum: 0.5
        expected_output: Hello world
      - type: faithfulness
        llm_settings: *llm_settings
        minimum: 0.5
      - type: hallucination
        llm_settings: *llm_settings
        minimum: 0.5
        context: ["Cardiology"]
    #end asserts
  #end test_case2
#end test_cases