Python API Reference

The main entrypoint for using bench is the TestSuite which is used to create new test suites, initiate test runs, and manage suite and run data.

• arthur_bench.client
  • Subpackages
    • arthur_bench.client.auth
      • Submodules
        • get_arthur_internal_user_org()
        • get_auth_info()
        • get_current_org()
        • user_login()
        • AuthRefresher
          • AuthRefresher.ALGORITHMS
          • AuthRefresher.AUTH_KEY
          • AuthRefresher.MINS_BEFORE_EXPIRY_TO_REFRESH
          • AuthRefresher.refresh()
    • arthur_bench.client.http
      • Submodules
        • construct_url()
        • HTTPClient
          • HTTPClient.delete()
          • HTTPClient.get()
          • HTTPClient.patch()
          • HTTPClient.post()
          • HTTPClient.put()
          • HTTPClient.send()
          • HTTPClient.set_path_prefix()
        • validate_multistatus_response_and_get_failures()
        • validate_response_status()
    • arthur_bench.client.local
      • Submodules
        • LocalBenchClient
          • LocalBenchClient.create_new_test_run()
          • LocalBenchClient.create_test_suite()
          • LocalBenchClient.delete_test_run()
          • LocalBenchClient.delete_test_suite()
          • LocalBenchClient.get_runs_for_test_suite()
          • LocalBenchClient.get_summary_statistics()
          • LocalBenchClient.get_test_run()
          • LocalBenchClient.get_test_suite()
          • LocalBenchClient.get_test_suite_by_name()
          • LocalBenchClient.get_test_suites()
        • PageInfo
          • PageInfo.end
          • PageInfo.page
          • PageInfo.page_size
          • PageInfo.sorted_pages
          • PageInfo.start
          • PageInfo.total_count
          • PageInfo.total_pages
    • arthur_bench.client.rest
      • Subpackages
        • arthur_bench.client.rest.admin
          • Submodules
            • ArthurAdminClient
              • ArthurAdminClient.authenticate()
              • ArthurAdminClient.get_current_user()
              • ArthurAdminClient.login()
            • AuthenticationInfo
              • AuthenticationInfo.email
              • AuthenticationInfo.external_user_id
              • AuthenticationInfo.first_name
              • AuthenticationInfo.internal_user_id
              • AuthenticationInfo.issuer
              • AuthenticationInfo.last_name
              • AuthenticationInfo.organization_ids
              • AuthenticationInfo.roles
              • AuthenticationInfo.service_account_id
              • AuthenticationInfo.username
            • LoginRequest
              • LoginRequest.login
              • LoginRequest.password
            • User
              • User.alert_notifications_enabled
              • User.created_at
              • User.email
              • User.first_name
              • User.help_mode_enabled
              • User.id
              • User.last_name
              • User.roles
              • User.show_intro_sequence
              • User.username
            • UserContext
              • UserContext.id
              • UserContext.name
            • UserResponse
              • UserResponse.alert_notifications_enabled
              • UserResponse.contexts
              • UserResponse.created_at
              • UserResponse.email
              • UserResponse.first_name
              • UserResponse.help_mode_enabled
              • UserResponse.id
              • UserResponse.last_name
              • UserResponse.organization_id
              • UserResponse.organization_name
              • UserResponse.plan
              • UserResponse.roles
              • UserResponse.show_intro_sequence
              • UserResponse.username
        • arthur_bench.client.rest.bench
          • Submodules
            • ArthurBenchClient
              • ArthurBenchClient.create_new_test_run()
              • ArthurBenchClient.create_test_suite()
              • ArthurBenchClient.delete_test_run()
              • ArthurBenchClient.delete_test_suite()
              • ArthurBenchClient.get_runs_for_test_suite()
              • ArthurBenchClient.get_summary_statistics()
              • ArthurBenchClient.get_test_run()
              • ArthurBenchClient.get_test_suite()
              • ArthurBenchClient.get_test_suites()
              • ArthurBenchClient.score_hallucination()
      • Submodules
        • ArthurClient
  • Submodules
    • BenchClient
      • BenchClient.check_run_exists()
      • BenchClient.create_new_test_run()
      • BenchClient.create_test_suite()
      • BenchClient.delete_test_run()
      • BenchClient.delete_test_suite()
      • BenchClient.get_runs_for_test_suite()
      • BenchClient.get_suite_if_exists()
      • BenchClient.get_summary_statistics()
      • BenchClient.get_test_run()
      • BenchClient.get_test_suite()
      • BenchClient.get_test_suites()
• arthur_bench.client.auth
  • Submodules
    • get_arthur_internal_user_org()
    • get_auth_info()
    • get_current_org()
    • user_login()
    • AuthRefresher
      • AuthRefresher.ALGORITHMS
      • AuthRefresher.AUTH_KEY
      • AuthRefresher.MINS_BEFORE_EXPIRY_TO_REFRESH
      • AuthRefresher.refresh()
• arthur_bench.client.http
  • Submodules
    • construct_url()
    • HTTPClient
      • HTTPClient.delete()
      • HTTPClient.get()
      • HTTPClient.patch()
      • HTTPClient.post()
      • HTTPClient.put()
      • HTTPClient.send()
      • HTTPClient.set_path_prefix()
    • validate_multistatus_response_and_get_failures()
    • validate_response_status()
• arthur_bench.client.local
  • Submodules
    • LocalBenchClient
      • LocalBenchClient.create_new_test_run()
      • LocalBenchClient.create_test_suite()
      • LocalBenchClient.delete_test_run()
      • LocalBenchClient.delete_test_suite()
      • LocalBenchClient.get_runs_for_test_suite()
      • LocalBenchClient.get_summary_statistics()
      • LocalBenchClient.get_test_run()
      • LocalBenchClient.get_test_suite()
      • LocalBenchClient.get_test_suite_by_name()
      • LocalBenchClient.get_test_suites()
    • PageInfo
      • PageInfo.end
      • PageInfo.page
      • PageInfo.page_size
      • PageInfo.sorted_pages
      • PageInfo.start
      • PageInfo.total_count
      • PageInfo.total_pages
• arthur_bench.client.rest
  • Subpackages
    • arthur_bench.client.rest.admin
      • Submodules
        • ArthurAdminClient
          • ArthurAdminClient.authenticate()
          • ArthurAdminClient.get_current_user()
          • ArthurAdminClient.login()
        • AuthenticationInfo
          • AuthenticationInfo.email
          • AuthenticationInfo.external_user_id
          • AuthenticationInfo.first_name
          • AuthenticationInfo.internal_user_id
          • AuthenticationInfo.issuer
          • AuthenticationInfo.last_name
          • AuthenticationInfo.organization_ids
          • AuthenticationInfo.roles
          • AuthenticationInfo.service_account_id
          • AuthenticationInfo.username
        • LoginRequest
          • LoginRequest.login
          • LoginRequest.password
        • User
          • User.alert_notifications_enabled
          • User.created_at
          • User.email
          • User.first_name
          • User.help_mode_enabled
          • User.id
          • User.last_name
          • User.roles
          • User.show_intro_sequence
          • User.username
        • UserContext
          • UserContext.id
          • UserContext.name
        • UserResponse
          • UserResponse.alert_notifications_enabled
          • UserResponse.contexts
          • UserResponse.created_at
          • UserResponse.email
          • UserResponse.first_name
          • UserResponse.help_mode_enabled
          • UserResponse.id
          • UserResponse.last_name
          • UserResponse.organization_id
          • UserResponse.organization_name
          • UserResponse.plan
          • UserResponse.roles
          • UserResponse.show_intro_sequence
          • UserResponse.username
    • arthur_bench.client.rest.bench
      • Submodules
        • ArthurBenchClient
          • ArthurBenchClient.create_new_test_run()
          • ArthurBenchClient.create_test_suite()
          • ArthurBenchClient.delete_test_run()
          • ArthurBenchClient.delete_test_suite()
          • ArthurBenchClient.get_runs_for_test_suite()
          • ArthurBenchClient.get_summary_statistics()
          • ArthurBenchClient.get_test_run()
          • ArthurBenchClient.get_test_suite()
          • ArthurBenchClient.get_test_suites()
          • ArthurBenchClient.score_hallucination()
  • Submodules
    • ArthurClient
• arthur_bench.client.rest.admin
  • Submodules
    • ArthurAdminClient
      • ArthurAdminClient.authenticate()
      • ArthurAdminClient.get_current_user()
      • ArthurAdminClient.login()
    • AuthenticationInfo
      • AuthenticationInfo.email
      • AuthenticationInfo.external_user_id
      • AuthenticationInfo.first_name
      • AuthenticationInfo.internal_user_id
      • AuthenticationInfo.issuer
      • AuthenticationInfo.last_name
      • AuthenticationInfo.organization_ids
      • AuthenticationInfo.roles
      • AuthenticationInfo.service_account_id
      • AuthenticationInfo.username
    • LoginRequest
      • LoginRequest.login
      • LoginRequest.password
    • User
      • User.alert_notifications_enabled
      • User.created_at
      • User.email
      • User.first_name
      • User.help_mode_enabled
      • User.id
      • User.last_name
      • User.roles
      • User.show_intro_sequence
      • User.username
    • UserContext
      • UserContext.id
      • UserContext.name
    • UserResponse
      • UserResponse.alert_notifications_enabled
      • UserResponse.contexts
      • UserResponse.created_at
      • UserResponse.email
      • UserResponse.first_name
      • UserResponse.help_mode_enabled
      • UserResponse.id
      • UserResponse.last_name
      • UserResponse.organization_id
      • UserResponse.organization_name
      • UserResponse.plan
      • UserResponse.roles
      • UserResponse.show_intro_sequence
      • UserResponse.username
• arthur_bench.client.rest.bench
  • Submodules
    • ArthurBenchClient
      • ArthurBenchClient.create_new_test_run()
      • ArthurBenchClient.create_test_suite()
      • ArthurBenchClient.delete_test_run()
      • ArthurBenchClient.delete_test_suite()
      • ArthurBenchClient.get_runs_for_test_suite()
      • ArthurBenchClient.get_summary_statistics()
      • ArthurBenchClient.get_test_run()
      • ArthurBenchClient.get_test_suite()
      • ArthurBenchClient.get_test_suites()
      • ArthurBenchClient.score_hallucination()
• arthur_bench.exceptions
  • Submodules
    • ArthurError
    • ArthurInternalError
    • ArthurUserError
    • ExpectedParameterNotFoundError
    • ForbiddenError
    • InternalTypeError
    • InternalValueError
    • MethodNotApplicableError
    • MissingParameterError
    • NotFoundError
    • PaymentRequiredError
    • ResponseClientError
    • ResponseRedirectError
    • ResponseServerError
    • UnauthorizedError
    • UserTypeError
    • UserValueError
    • arthur_excepted()
• arthur_bench.models
  • Submodules
    • CategoricalHistogramItem
      • CategoricalHistogramItem.category
      • CategoricalHistogramItem.count
    • Category
      • Category.description
      • Category.name
    • CommonSortEnum
      • CommonSortEnum.CREATED_AT_ASC
      • CommonSortEnum.CREATED_AT_DESC
      • CommonSortEnum.NAME_ASC
      • CommonSortEnum.NAME_DESC
    • CreateRunRequest
      • CreateRunRequest.Config
        • CreateRunRequest.Config.allow_population_by_field_name
      • CreateRunRequest.consistent_categories()
      • CreateRunRequest.description
      • CreateRunRequest.foundation_model
      • CreateRunRequest.model_name
      • CreateRunRequest.model_version
      • CreateRunRequest.name
      • CreateRunRequest.prompt_template
      • CreateRunRequest.test_cases
    • CreateRunResponse
      • CreateRunResponse.id
    • HistogramItem
      • HistogramItem.count
      • HistogramItem.high
      • HistogramItem.low
    • PaginatedRun
      • PaginatedRun.Config
        • PaginatedRun.Config.allow_population_by_field_name
      • PaginatedRun.created_at
      • PaginatedRun.id
      • PaginatedRun.name
      • PaginatedRun.page
      • PaginatedRun.page_size
      • PaginatedRun.test_cases
      • PaginatedRun.test_suite_id
      • PaginatedRun.total_count
      • PaginatedRun.total_pages
      • PaginatedRun.updated_at
    • PaginatedRuns
      • PaginatedRuns.page
      • PaginatedRuns.page_size
      • PaginatedRuns.test_runs
      • PaginatedRuns.total_count
      • PaginatedRuns.total_pages
    • PaginatedTestSuite
      • PaginatedTestSuite.created_at
      • PaginatedTestSuite.description
      • PaginatedTestSuite.id
      • PaginatedTestSuite.last_run_time
      • PaginatedTestSuite.name
      • PaginatedTestSuite.num_runs
      • PaginatedTestSuite.page
      • PaginatedTestSuite.page_size
      • PaginatedTestSuite.scoring_method
      • PaginatedTestSuite.test_cases
      • PaginatedTestSuite.total_count
      • PaginatedTestSuite.total_pages
      • PaginatedTestSuite.updated_at
    • PaginatedTestSuites
      • PaginatedTestSuites.page
      • PaginatedTestSuites.page_size
      • PaginatedTestSuites.test_suites
      • PaginatedTestSuites.total_count
      • PaginatedTestSuites.total_pages
    • RunResult
      • RunResult.id
      • RunResult.input
      • RunResult.output
      • RunResult.reference_output
      • RunResult.score
      • RunResult.score_result
      • RunResult.score_result_backwards_compatible()
    • ScoreResult
      • ScoreResult.category
      • ScoreResult.contains_score()
      • ScoreResult.score
    • ScorerOutputType
      • ScorerOutputType.Categorical
      • ScorerOutputType.Continuous
    • ScoringMethod
      • ScoringMethod.categories
      • ScoringMethod.config
      • ScoringMethod.name
      • ScoringMethod.output_type
      • ScoringMethod.scoring_method_categorical_defined()
      • ScoringMethod.type
    • ScoringMethodType
      • ScoringMethodType.BuiltIn
      • ScoringMethodType.Custom
    • SummaryItem
      • SummaryItem.avg_score
      • SummaryItem.either_continuous_or_categorical()
      • SummaryItem.histogram
      • SummaryItem.id
      • SummaryItem.name
    • TestCaseOutput
      • TestCaseOutput.id
      • TestCaseOutput.output
      • TestCaseOutput.score
      • TestCaseOutput.score_result
      • TestCaseOutput.score_result_backwards_compatible()
    • TestCaseRequest
      • TestCaseRequest.input
      • TestCaseRequest.reference_output
    • TestCaseResponse
      • TestCaseResponse.id
      • TestCaseResponse.input
      • TestCaseResponse.reference_output
    • TestCaseSortEnum
      • TestCaseSortEnum.SCORE_ASC
      • TestCaseSortEnum.SCORE_DESC
    • TestRunMetadata
      • TestRunMetadata.avg_score
      • TestRunMetadata.created_at
      • TestRunMetadata.id
      • TestRunMetadata.model_version
      • TestRunMetadata.name
      • TestRunMetadata.prompt_template
      • TestRunMetadata.updated_at
    • TestRunSortEnum
      • TestRunSortEnum.AVG_SCORE_ASC
      • TestRunSortEnum.AVG_SCORE_DESC
    • TestSuiteMetadata
      • TestSuiteMetadata.created_at
      • TestSuiteMetadata.description
      • TestSuiteMetadata.id
      • TestSuiteMetadata.last_run_time
      • TestSuiteMetadata.name
      • TestSuiteMetadata.scoring_method
      • TestSuiteMetadata.updated_at
    • TestSuiteRequest
      • TestSuiteRequest.description
      • TestSuiteRequest.name
      • TestSuiteRequest.null_reference_outputs_all_or_none()
      • TestSuiteRequest.scoring_method
      • TestSuiteRequest.scoring_method_backwards_compatible()
      • TestSuiteRequest.test_cases
    • TestSuiteSortEnum
      • TestSuiteSortEnum.LAST_RUNTIME_ASC
      • TestSuiteSortEnum.LAST_RUNTIME_DESC
    • TestSuiteSummary
      • TestSuiteSummary.categorical
      • TestSuiteSummary.num_test_cases
      • TestSuiteSummary.page
      • TestSuiteSummary.page_size
      • TestSuiteSummary.summary
      • TestSuiteSummary.total_count
      • TestSuiteSummary.total_pages
    • HallucinationScoreRequest
      • HallucinationScoreRequest.context
      • HallucinationScoreRequest.response
    • HallucinationScoreResponse
      • HallucinationScoreResponse.hallucination
      • HallucinationScoreResponse.reason
• arthur_bench.run
  • Submodules
    • TestRun
      • TestRun.Config
        • TestRun.Config.arbitrary_types_allowed
      • TestRun.categories
      • TestRun.client
      • TestRun.from_flattened()
      • TestRun.id
      • TestRun.output
      • TestRun.save()
      • TestRun.scores
      • TestRun.test_suite_id
    • TestSuite
      • TestSuite.arun()
      • TestSuite.description
      • TestSuite.input_texts
      • TestSuite.name
      • TestSuite.reference_outputs
      • TestSuite.run()
      • TestSuite.save()
      • TestSuite.scoring_method
      • TestSuite.test_cases
• arthur_bench.scoring
  • ScoringMethodName
    • ScoringMethodName.BERTScore
    • ScoringMethodName.ExactMatch
    • ScoringMethodName.Hallucination
    • ScoringMethodName.HedgingLanguage
    • ScoringMethodName.PythonUnitTesting
    • ScoringMethodName.QACorrectness
    • ScoringMethodName.Readability
    • ScoringMethodName.Specificity
    • ScoringMethodName.SummaryQuality
    • ScoringMethodName.WordCountMatch
  • scorer_from_string()
  • Submodules
    • BERTScore
      • BERTScore.name()
      • BERTScore.run_batch()
      • BERTScore.to_dict()
    • ExactMatch
      • ExactMatch.categories()
      • ExactMatch.is_categorical()
      • ExactMatch.name()
      • ExactMatch.run_batch()
    • Hallucination
      • Hallucination.categories()
      • Hallucination.is_categorical()
      • Hallucination.name()
      • Hallucination.requires_reference()
      • Hallucination.run_batch()
      • Hallucination.to_dict()
    • HedgingLanguage
      • HedgingLanguage.name()
      • HedgingLanguage.requires_reference()
      • HedgingLanguage.run_batch()
      • HedgingLanguage.to_dict()
    • PythonUnitTesting
      • PythonUnitTesting.categories()
      • PythonUnitTesting.is_categorical()
      • PythonUnitTesting.name()
      • PythonUnitTesting.requires_reference()
      • PythonUnitTesting.run()
      • PythonUnitTesting.run_batch()
      • PythonUnitTesting.to_dict()
    • QAQualityCorrectness
      • QAQualityCorrectness.arun_batch()
      • QAQualityCorrectness.categories()
      • QAQualityCorrectness.is_categorical()
      • QAQualityCorrectness.name()
      • QAQualityCorrectness.requires_reference()
      • QAQualityCorrectness.run_batch()
      • QAQualityCorrectness.to_dict()
      • QAQualityCorrectness.validate_batch()
    • Readability
      • Readability.name()
      • Readability.requires_reference()
      • Readability.run_batch()
    • Scorer
      • Scorer.arun()
      • Scorer.arun_batch()
      • Scorer.categories()
      • Scorer.from_dict()
      • Scorer.is_categorical()
      • Scorer.name()
      • Scorer.requires_reference()
      • Scorer.run()
      • Scorer.run_batch()
      • Scorer.to_dict()
      • Scorer.to_metadata()
      • Scorer.type()
    • Specificity
      • Specificity.get_mean_word_freq()
      • Specificity.get_num_vague_words()
      • Specificity.get_pn_and_num()
      • Specificity.name()
      • Specificity.requires_reference()
      • Specificity.run_batch()
    • SummaryQuality
      • SummaryQuality.arun()
      • SummaryQuality.arun_batch()
      • SummaryQuality.categories()
      • SummaryQuality.is_categorical()
      • SummaryQuality.name()
      • SummaryQuality.run()
      • SummaryQuality.run_batch()
      • SummaryQuality.to_dict()
      • SummaryQuality.validate_batch()
    • truncate_input_text()
    • suppress_warnings
    • WordCountMatch
      • WordCountMatch.name()
      • WordCountMatch.run_batch()
• arthur_bench.server
  • Submodules
• arthur_bench.telemetry
  • Submodules
• arthur_bench.utils
  • Submodules
    • get_file_extension()
    • load_suite_from_csv()
    • load_suite_from_dataframe()
    • load_suite_from_json()
    • load_suite_from_list()