CrucibleIR.Builder (CrucibleIR v0.3.0)

Fluent builder API for constructing experiments.

The Builder module provides a fluent, chainable API for building complex CrucibleIR experiments with validation. It simplifies the creation of experiments by providing convenience methods and automatic validation.

Example

{:ok, exp} =
  CrucibleIR.Builder.experiment(:my_exp)
  |> CrucibleIR.Builder.with_description("Test experiment")
  |> CrucibleIR.Builder.with_backend(:gpt4)
  |> CrucibleIR.Builder.add_stage(:inference)
  |> CrucibleIR.Builder.with_ensemble(:majority)
  |> CrucibleIR.Builder.with_stats([:ttest], alpha: 0.01)
  |> CrucibleIR.Builder.build()

Fluent API

All builder functions return the modified experiment struct, allowing for method chaining. Use build/1 at the end to validate and finalize.

Summary

Types

builder_experiment()

Functions

add_output(exp, name, opts \\ [])

Adds an output specification to the experiment.

add_stage(exp, stage_name, opts \\ [])

Adds a stage to the experiment pipeline.

build(exp)

Validates and finalizes the experiment.

experiment(id)

Creates a new experiment builder with the given ID.

with_backend(exp, backend_id, opts \\ [])

Adds a backend to the experiment.

with_baseline(exp, model_id, opts \\ [])

Sets a baseline model for comparison experiments.

with_dataset(exp, dataset_name, opts \\ [])

Adds a dataset to the experiment.

with_description(exp, description)

Adds a description to the experiment.

with_ensemble(exp, strategy, opts \\ [])

Adds ensemble voting configuration to the experiment.

with_experiment_type(exp, type)

Sets the experiment type.

with_fairness(exp, opts \\ [])

Adds fairness checking configuration to the experiment.

with_feedback(exp, opts \\ [])

Adds feedback collection configuration to the experiment.

with_guardrails(exp, opts \\ [])

Adds guardrails configuration to the experiment.

with_hedging(exp, strategy, opts \\ [])

Adds request hedging configuration to the experiment.

with_model_version(exp, model_id, version, opts \\ [])

Sets a model version for the experiment.

with_stats(exp, tests, opts \\ [])

Adds statistical testing configuration to the experiment.

with_training_config(exp, model_ref, dataset_ref, opts \\ [])

Adds training configuration to the experiment.

Types

builder_experiment()

@type builder_experiment() :: %CrucibleIR.Experiment{
  backend: CrucibleIR.BackendRef.t() | nil,
  baseline: term(),
  created_at: DateTime.t() | nil,
  dataset: CrucibleIR.DatasetRef.t() | nil,
  description: String.t() | nil,
  experiment_type: term(),
  id: atom(),
  metadata: map() | nil,
  model_version: term(),
  outputs: [CrucibleIR.OutputSpec.t()] | nil,
  owner: String.t() | nil,
  pipeline: [CrucibleIR.StageDef.t()],
  reliability: CrucibleIR.Reliability.Config.t() | nil,
  tags: [atom()] | nil,
  training_config: term(),
  updated_at: DateTime.t() | nil
}

Functions

add_output(exp, name, opts \\ [])

@spec add_output(builder_experiment(), atom(), keyword()) :: builder_experiment()

Adds an output specification to the experiment.

Parameters

exp - Experiment struct
name - Atom name for the output
opts - Optional keyword list with :formats, :sink, :options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.add_output(:results)
iex> hd(exp.outputs).name
:results

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.add_output(:results, formats: [:json, :html])
iex> hd(exp.outputs).formats
[:json, :html]

add_stage(exp, stage_name, opts \\ [])

@spec add_stage(builder_experiment(), atom(), keyword()) :: builder_experiment()

Adds a stage to the experiment pipeline.

Parameters

exp - Experiment struct
stage_name - Atom name for the stage
opts - Optional keyword list with :module, :options, :enabled

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.add_stage(:preprocessing)
iex> hd(exp.pipeline).name
:preprocessing

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.add_stage(:preprocessing, enabled: false)
iex> hd(exp.pipeline).enabled
false

build(exp)

@spec build(builder_experiment()) ::
  {:ok, CrucibleIR.Experiment.t()} | {:error, [String.t()]}

Validates and finalizes the experiment.

Returns {:ok, experiment} if valid, or {:error, errors} if invalid.

Examples

iex> {:ok, exp} = CrucibleIR.Builder.experiment(:test)
...> |> CrucibleIR.Builder.with_backend(:gpt4)
...> |> CrucibleIR.Builder.add_stage(:run)
...> |> CrucibleIR.Builder.build()
iex> exp.id
:test

iex> {:error, errors} = CrucibleIR.Builder.experiment(:test)
...> |> CrucibleIR.Builder.build()
iex> is_list(errors)
true

experiment(id)

@spec experiment(atom()) :: builder_experiment()

Creates a new experiment builder with the given ID.

Parameters

id - Atom identifier for the experiment

Returns

An Experiment struct with only the id set (incomplete, not validated).

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> exp.id
:test

with_backend(exp, backend_id, opts \\ [])

@spec with_backend(builder_experiment(), atom(), keyword()) :: builder_experiment()

Adds a backend to the experiment.

Parameters

exp - Experiment struct
backend_id - Atom identifier for the backend
opts - Optional keyword list with :profile and :options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_backend(:gpt4)
iex> exp.backend.id
:gpt4

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_backend(:gpt4, profile: :fast, options: %{temp: 0.7})
iex> exp.backend.profile
:fast

with_baseline(exp, model_id, opts \\ [])

@spec with_baseline(builder_experiment(), atom(), keyword()) :: builder_experiment()

Sets a baseline model for comparison experiments.

Parameters

exp - Experiment struct
model_id - Baseline model identifier
opts - Optional keyword list with model options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_baseline(:gpt3)
iex> exp.baseline.id
:gpt3

with_dataset(exp, dataset_name, opts \\ [])

@spec with_dataset(builder_experiment(), atom() | String.t(), keyword()) ::
  builder_experiment()

Adds a dataset to the experiment.

Parameters

exp - Experiment struct
dataset_name - Atom or string name for the dataset
opts - Optional keyword list with :provider, :split, :options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_dataset(:mmlu)
iex> exp.dataset.name
:mmlu

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_dataset(:mmlu, split: :test)
iex> exp.dataset.split
:test

with_description(exp, description)

@spec with_description(builder_experiment(), String.t()) :: builder_experiment()

Adds a description to the experiment.

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_description("My experiment")
iex> exp.description
"My experiment"

with_ensemble(exp, strategy, opts \\ [])

@spec with_ensemble(builder_experiment(), atom(), keyword()) :: builder_experiment()

Adds ensemble voting configuration to the experiment.

Parameters

exp - Experiment struct
strategy - Voting strategy atom
opts - Optional keyword list with ensemble options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_ensemble(:majority)
iex> exp.reliability.ensemble.strategy
:majority

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_ensemble(:weighted, models: [:gpt4, :claude])
iex> exp.reliability.ensemble.models
[:gpt4, :claude]

with_experiment_type(exp, type)

@spec with_experiment_type(builder_experiment(), atom()) :: builder_experiment()

Sets the experiment type.

Parameters

exp - Experiment struct
type - Experiment type (:evaluation, :training, :comparison, :ablation)

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_experiment_type(:evaluation)
iex> exp.experiment_type
:evaluation

with_fairness(exp, opts \\ [])

@spec with_fairness(
  builder_experiment(),
  keyword()
) :: builder_experiment()

Adds fairness checking configuration to the experiment.

Parameters

exp - Experiment struct
opts - Keyword list with fairness options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_fairness(metrics: [:demographic_parity])
iex> exp.reliability.fairness.metrics
[:demographic_parity]

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_fairness(threshold: 0.8, enabled: true)
iex> exp.reliability.fairness.threshold
0.8

with_feedback(exp, opts \\ [])

@spec with_feedback(
  builder_experiment(),
  keyword()
) :: builder_experiment()

Adds feedback collection configuration to the experiment.

Parameters

exp - Experiment struct
opts - Keyword list with feedback options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_feedback(enabled: true, sampling_rate: 0.1)
iex> exp.reliability.feedback.sampling_rate
0.1

with_guardrails(exp, opts \\ [])

@spec with_guardrails(
  builder_experiment(),
  keyword()
) :: builder_experiment()

Adds guardrails configuration to the experiment.

Parameters

exp - Experiment struct
opts - Keyword list with guardrail options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_guardrails(profiles: [:strict])
iex> exp.reliability.guardrails.profiles
[:strict]

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_guardrails(pii_detection: true)
iex> exp.reliability.guardrails.pii_detection
true

with_hedging(exp, strategy, opts \\ [])

@spec with_hedging(builder_experiment(), atom(), keyword()) :: builder_experiment()

Adds request hedging configuration to the experiment.

Parameters

exp - Experiment struct
strategy - Hedging strategy atom
opts - Optional keyword list with hedging options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_hedging(:fixed)
iex> exp.reliability.hedging.strategy
:fixed

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_hedging(:percentile, delay_ms: 100)
iex> exp.reliability.hedging.delay_ms
100

with_model_version(exp, model_id, version, opts \\ [])

@spec with_model_version(builder_experiment(), atom(), String.t(), keyword()) ::
  builder_experiment()

Sets a model version for the experiment.

Parameters

exp - Experiment struct
model_id - Model identifier
version - Version string
opts - Optional keyword list with version options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_model_version(:gpt2, "1.0.0")
iex> exp.model_version.version
"1.0.0"

with_stats(exp, tests, opts \\ [])

@spec with_stats(builder_experiment(), [atom()], keyword()) :: builder_experiment()

Adds statistical testing configuration to the experiment.

Parameters

exp - Experiment struct
tests - List of statistical test atoms
opts - Optional keyword list with stats options

Examples

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_stats([:ttest])
iex> exp.reliability.stats.tests
[:ttest]

iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_stats([:ttest, :bootstrap], alpha: 0.01)
iex> exp.reliability.stats.alpha
0.01

with_training_config(exp, model_ref, dataset_ref, opts \\ [])

@spec with_training_config(
  builder_experiment(),
  CrucibleIR.ModelRef.t(),
  CrucibleIR.DatasetRef.t(),
  keyword()
) :: builder_experiment()

Adds training configuration to the experiment.

Parameters

exp - Experiment struct
model_ref - ModelRef struct or keyword to create one
dataset_ref - DatasetRef struct or keyword to create one
opts - Optional keyword list with training options

Examples

iex> model = %CrucibleIR.ModelRef{id: :gpt2}
iex> dataset = %CrucibleIR.DatasetRef{name: :wikitext}
iex> exp = CrucibleIR.Builder.experiment(:test)
iex> |> CrucibleIR.Builder.with_training_config(model, dataset, epochs: 10)
iex> exp.training_config.epochs
10