class RubyLLM::Evaluation::Trial

The recorded outcome of one case and repetition, including failed attempts.

Attributes

assertion_count [R]

Returns the number of Ruby assertions attempted.

assertion_failure [R]

Returns an assertion failure, or nil.

duration [R]

Returns elapsed wall-clock seconds, including evaluators.

error [R]

Returns an execution or evidence-conversion error, or nil.

evaluations [R]

Returns each criterion’s Evaluation::Result.

evidence [R]

Returns the frozen evidence supplied to evaluators.

output [R]

Returns the primary output extracted from the returned value.

repetition [R]

Returns the one-based repetition number.

result [R]

Returns the original value returned by perform, when it completed.

test_case [R]

Returns the case whose inputs were executed.

Public Instance Methods

cost ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 48
def cost
  ruby_llm_usage_cost
end

Returns a Cost aggregate for this trial, including task and evaluator requests.

evaluator_cost ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 68
def evaluator_cost
  Cost.aggregate(@evaluator_usage.map(&:cost), complete: @evaluator_usage.all?(&:cost_available?))
end

Returns the cost of the evaluators and their tools.

evaluator_tokens ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 58
def evaluator_tokens
  Tokens.aggregate(@evaluator_usage.map(&:tokens))
end

Returns the tokens used by the evaluators and their tools.

passed? ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 82
def passed?
  status == :passed
end

Returns whether every assessment passed and no error occurred.

status ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 73
def status
  statuses = evaluations.map(&:status)
  statuses << :error if error
  statuses << :failed if assertion_failure
  statuses << :measured if evaluations.empty? && assertion_count.to_i.zero?
  %i[error failed unassessed measured].find { |status| statuses.include?(status) } || :passed
end

Returns :passed, :failed, :measured, :unassessed, or :error. Empty trials are never passes.

task_cost ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 63
def task_cost
  Cost.aggregate(@task_usage.map(&:cost), complete: @task_usage.all?(&:cost_available?))
end

Returns the task’s cost, including setup, assertions, and teardown requests.

task_tokens ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 53
def task_tokens
  Tokens.aggregate(@task_usage.map(&:tokens))
end

Returns the task’s tokens, including setup, assertions, and teardown requests.

to_h ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 87
def to_h
  { case: test_case.to_h, repetition:, status:, evidence:, evaluations: evaluations.map(&:to_h),
    assertion_count:, assertion_failure: assertion_failure&.message, duration:,
    error: error && { class: error.class.name, message: error.message },
    tokens: tokens.to_h, cost: cost.to_h,
    task_tokens: task_tokens.to_h, evaluator_tokens: evaluator_tokens.to_h,
    task_cost: task_cost.to_h, evaluator_cost: evaluator_cost.to_h }
end

Returns the portable trial record, excluding the live application object.

tokens ()

Source

# File lib/ruby_llm/evaluation/trial.rb, line 43
def tokens
  ruby_llm_usage_tokens
end

Returns a Tokens aggregate for every provider attempt made during this trial.