class RubyLLM::Judge

Defines reusable probability, choice, and score questions over application data.

class TicketTriage < RubyLLM::Judge
  model "jev-latest"
  probability :urgent, "Does this need attention today?"
  choice :department, "Which team should handle this?" do
    billing "Payments and refunds"
    technical "Bugs and integrations"
  end
end

TicketTriage.judge("Please refund my duplicate charge today.")[:urgent].probability

Blocks and procs resolve once per judgment, with declared inputs available as methods. Hashes and arrays preserve structured instructions and criteria.

Public Class Methods

choice (name, instructions = nil, options = nil, &)

Source

# File lib/ruby_llm/judge.rb, line 78
def choice(name, instructions = nil, options = nil, &)
  declare_question(name, :choice, instructions, options, &)
end

Declares a question selecting one named option. Options are a Hash, a proc returning a Hash, or a block declaring named descriptions.

choice :department, "Which team?" do
  billing "Payments and refunds"
  other nil
end

inputs (*names)

Source

# File lib/ruby_llm/judge.rb, line 44
def inputs(*names)
  return @input_names || [] if names.empty?

  @input_names = names.flatten.map(&:to_sym).freeze
end

Declares required runtime inputs available in question and data blocks. With no arguments, returns the declared names.

inputs :teams
choice :department, "Which team?", -> { teams.to_h { |team| [team.slug, team.description] } }

judge (input = nil, **options, &)

Source

# File lib/ruby_llm/judge.rb, line 97
def judge(input = nil, **options, &)
  values = options.slice(*inputs)
  new(**values).judge(input, **options.except(*inputs), &)
end

Judges text, a Hash, or an Array and returns a Judgment. A block or proc can supply the input. Declared inputs are accepted as keyword arguments; remaining options are forwarded to judge.

TicketTriage.judge do
  message "Please refund the duplicate charge today."
end

model (value = nil, **options, &block)

Source

# File lib/ruby_llm/judge.rb, line 31
def model(value = nil, **options, &block)
  return @model_options || {} if value.nil? && options.empty? && !block

  raise ArgumentError, 'Pass a model or a block, not both' if !value.nil? && block

  @model_options = options.merge(model: block || value).freeze
end

Sets the model and provider used by this judge. A block or proc resolves the model using runtime inputs. With no arguments, returns the settings.

new (**inputs)

Source

# File lib/ruby_llm/judge.rb, line 120
def initialize(**inputs)
  missing = self.class.inputs - inputs.keys
  extra = inputs.keys - self.class.inputs
  raise ArgumentError, "Missing judge inputs: #{missing.join(', ')}" unless missing.empty?
  raise ArgumentError, "Unknown judge inputs: #{extra.join(', ')}" unless extra.empty?

  inputs.each do |name, value|
    raise ArgumentError, "Judge input conflicts with a method: #{name}" if respond_to?(name, true)

    define_singleton_method(name) { value }
  end
end

Creates a judge with the runtime inputs declared by its class.

probability (name, instructions = nil, criteria = nil, &)

Source

# File lib/ruby_llm/judge.rb, line 67
def probability(name, instructions = nil, criteria = nil, &)
  declare_question(name, :probability, instructions, criteria, &)
end

Declares a yes/no question whose answer is the probability of yes. Optional criteria describe yes and no in a Hash, proc, or block.

probability :urgent, "Does this need attention today?" do
  yes "An explicit deadline today"
  no "No deadline or a later deadline"
end

provider_options (value = nil, &block)

Source

# File lib/ruby_llm/judge.rb, line 52
def provider_options(value = nil, &block)
  return @provider_options || {} if value.nil? && !block

  raise ArgumentError, 'Pass provider options or a block, not both' if !value.nil? && block

  @provider_options = Data.copy(block || value)
end

Sets provider-specific request options as a Hash, proc, or block. With no arguments, returns the declared options.

score (name, instructions = nil, levels = nil, &)

Source

# File lib/ruby_llm/judge.rb, line 86
def score(name, instructions = nil, levels = nil, &)
  declare_question(name, :score, instructions, levels, &)
end

Declares a question scored against ordered levels. Levels are an Array or a proc or block returning one. Scores can fall between level indexes.

score :frustration, "How frustrated is the customer?", ["Calm", "Frustrated", "Angry"]

Public Instance Methods

judge (input = nil, questions: {}, context: nil, metadata: nil, **options, &block)

Source

# File lib/ruby_llm/judge.rb, line 141
def judge(input = nil, questions: {}, context: nil, metadata: nil, **options, &block)
  raise ArgumentError, 'Pass judgment input or a block, not both' if !input.nil? && block

  data = resolve_data(block || input)
  unless data.is_a?(String) || data.is_a?(Hash) || data.is_a?(Array)
    raise ArgumentError, 'Judgment input must be text, a Hash, or an Array'
  end

  definitions = resolve_questions(questions)
  settings = self.class.model.merge(provider_options: self.class.provider_options).merge(options)
  settings = settings.transform_values { |value| resolve_data(value) }
  Judgment.judge(data, questions: definitions, context:, metadata:, **settings)
end

Judges the supplied input and returns a Judgment. Accepts model and provider overrides, an isolated context:, provider_options:, and instrumentation metadata:. Additional questions: are a Hash keyed by question name, with type:, instructions:, and criteria: (probability), options: (choice), or levels: (score). The block supplies input only.

RubyLLM.judge("Please help today", model: "jev-latest",
  questions: { urgent: { type: :probability, instructions: "Is this urgent?" } })