From bfc2875d62ba1efc94a5e794c241e13f8f67105c Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Tue, 5 May 2026 09:57:05 -0600 Subject: [PATCH] feat(traits,algorithms): add PartialProblem trait and Hyperband MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-fidelity optimization. Hyperband (Li et al. 2017) and its foundation Successive Halving (Karnin et al. 2013) tune hyperparameters by allocating *uneven* compute across configurations: sample many cheap-to-evaluate-at-low-budget configs, then promote the survivors to higher budgets. Crucial for ML hyperparameter tuning where each evaluation is a partial training run. This requires a new trait — `Problem::evaluate` is a single-shot black box, but Hyperband needs to evaluate the SAME decision at different fidelity budgets: pub trait PartialProblem { type Decision: Clone; fn objectives(&self) -> ObjectiveSpace; fn evaluate_at_budget(&self, decision: &Self::Decision, budget: f64) -> Evaluation; } `PartialProblem` is intentionally NOT a sub-trait of `Problem`. Implementors who already have a `Problem` and want their `evaluate_at_budget` to ignore budget can write a one-line wrapper. `Hyperband` is the optimizer: pub struct HyperbandConfig { max_budget: f64, eta: f64, max_brackets: usize, seed: u64, } pub struct Hyperband { config, initializer, ... } Single-objective only. The decision sampler is an `Initializer` so it works the same way as every other heuropt algorithm. Generic over decision type. --- src/algorithms/hyperband.rs | 286 ++++++++++++++++++++++++++++++++++++ src/algorithms/mod.rs | 2 + src/core/mod.rs | 2 + src/core/partial_problem.rs | 39 +++++ src/prelude.rs | 5 +- 5 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 src/algorithms/hyperband.rs create mode 100644 src/core/partial_problem.rs diff --git a/src/algorithms/hyperband.rs b/src/algorithms/hyperband.rs new file mode 100644 index 0000000..e4e85c2 --- /dev/null +++ b/src/algorithms/hyperband.rs @@ -0,0 +1,286 @@ +//! `Hyperband` — Li et al. 2017 multi-fidelity hyperparameter optimizer +//! built on Successive Halving (Karnin et al. 2013). + +use crate::core::candidate::Candidate; +use crate::core::evaluation::Evaluation; +use crate::core::objective::Direction; +use crate::core::partial_problem::PartialProblem; +use crate::core::population::Population; +use crate::core::result::OptimizationResult; +use crate::core::rng::rng_from_seed; +use crate::traits::Initializer; + +/// Configuration for [`Hyperband`]. +#[derive(Debug, Clone)] +pub struct HyperbandConfig { + /// Maximum fidelity budget per configuration. Common units: epochs, + /// timesteps, simulation iterations. + pub max_budget: f64, + /// Reduction factor `η`. Each Successive-Halving round survives + /// `1/η` of configurations and promotes them to `η×` budget. Li + /// et al. recommend 3 (which gives smin=1) or 4 (slightly more + /// aggressive promotion). + pub eta: f64, + /// Maximum number of brackets. The standard formula is + /// `floor(log_η(max_budget)) + 1`; pass a larger value to allow + /// it, smaller to truncate. + pub max_brackets: usize, + /// Seed for the deterministic RNG used to sample configurations. + pub seed: u64, +} + +impl Default for HyperbandConfig { + fn default() -> Self { + Self { max_budget: 81.0, eta: 3.0, max_brackets: 5, seed: 42 } + } +} + +/// Hyperband: a budget-aware single-objective optimizer for problems +/// where each evaluation can be performed at a tunable *fidelity* +/// (e.g. an ML training run for `budget` epochs). +/// +/// Each "bracket" is a Successive-Halving sweep that starts with many +/// configurations at low budget and progressively promotes the top +/// `1/η` fraction to higher budgets, eliminating the rest. Hyperband +/// runs several brackets with different (configurations, budget) +/// trade-offs — early brackets favor exploration (many configs at +/// low budget), later brackets favor exploitation (fewer configs run +/// near the max budget). The single best result across all brackets +/// is returned. +pub struct Hyperband +where + D: Clone, + I: Initializer, +{ + /// Algorithm configuration. + pub config: HyperbandConfig, + /// Random configuration sampler (same trait used everywhere else). + pub initializer: I, + _marker: std::marker::PhantomData, +} + +impl Hyperband +where + D: Clone, + I: Initializer, +{ + /// Construct a `Hyperband`. + pub fn new(config: HyperbandConfig, initializer: I) -> Self { + Self { config, initializer, _marker: std::marker::PhantomData } + } + + /// Run Hyperband on a multi-fidelity problem, returning the standard + /// `OptimizationResult`. Single-objective only. + pub fn run

(&mut self, problem: &P) -> OptimizationResult + where + P: PartialProblem, + { + assert!(self.config.max_budget > 0.0, "Hyperband max_budget must be > 0"); + assert!(self.config.eta > 1.0, "Hyperband eta must be > 1"); + assert!(self.config.max_brackets >= 1, "Hyperband max_brackets must be >= 1"); + let objectives = problem.objectives(); + assert!( + objectives.is_single_objective(), + "Hyperband requires exactly one objective", + ); + let direction = objectives.objectives[0].direction; + let mut rng = rng_from_seed(self.config.seed); + + // Number of brackets s_max = floor(log_η(max_budget)). + let s_max = (self.config.max_budget.ln() / self.config.eta.ln()).floor() as i64; + let s_max = (s_max as usize).min(self.config.max_brackets); + + let mut total_evaluations = 0usize; + let mut total_iterations = 0usize; + let mut best_seen: Option> = None; + + // Brackets are indexed s = s_max, s_max - 1, ..., 0. + for s in (0..=s_max).rev() { + let s_f = s as f64; + let n = ((s_max as f64 + 1.0) / (s_f + 1.0) + * self.config.eta.powf(s_f)) + .ceil() as usize; + let r = self.config.max_budget / self.config.eta.powf(s_f); + + // Sample n configurations. + let mut configs: Vec = self.initializer.initialize(n, &mut rng); + // SH inner loop. + for i in 0..=s { + let n_i = (n as f64 / self.config.eta.powi(i as i32)).floor() as usize; + let r_i = r * self.config.eta.powi(i as i32); + if configs.is_empty() { + break; + } + let evals: Vec = configs + .iter() + .map(|c| problem.evaluate_at_budget(c, r_i)) + .collect(); + total_evaluations += configs.len(); + + // Track best. + for (cfg, e) in configs.iter().zip(evals.iter()) { + let beats = match &best_seen { + None => true, + Some(b) => better(e, &b.evaluation, direction), + }; + if beats { + best_seen = Some(Candidate::new(cfg.clone(), e.clone())); + } + } + total_iterations += 1; + + // Top n_{i+1} survive. + let next_size = (n_i / self.config.eta as usize).max(1); + if next_size >= configs.len() { + continue; + } + let mut order: Vec = (0..configs.len()).collect(); + order.sort_by(|&a, &b| compare(&evals[a], &evals[b], direction)); + let keep: std::collections::HashSet = + order.into_iter().take(next_size).collect(); + let new_configs: Vec = configs + .into_iter() + .enumerate() + .filter_map(|(idx, c)| if keep.contains(&idx) { Some(c) } else { None }) + .collect(); + configs = new_configs; + } + } + + let best = best_seen.expect("at least one bracket ran"); + let population = Population::new(vec![best.clone()]); + let front = vec![best.clone()]; + OptimizationResult::new( + population, + front, + Some(best), + total_evaluations, + total_iterations, + ) + } +} + +fn compare(a: &Evaluation, b: &Evaluation, direction: Direction) -> std::cmp::Ordering { + match (a.is_feasible(), b.is_feasible()) { + (true, false) => std::cmp::Ordering::Less, + (false, true) => std::cmp::Ordering::Greater, + (false, false) => a + .constraint_violation + .partial_cmp(&b.constraint_violation) + .unwrap_or(std::cmp::Ordering::Equal), + (true, true) => match direction { + Direction::Minimize => a.objectives[0] + .partial_cmp(&b.objectives[0]) + .unwrap_or(std::cmp::Ordering::Equal), + Direction::Maximize => b.objectives[0] + .partial_cmp(&a.objectives[0]) + .unwrap_or(std::cmp::Ordering::Equal), + }, + } +} + +fn better(a: &Evaluation, b: &Evaluation, direction: Direction) -> bool { + compare(a, b, direction) == std::cmp::Ordering::Less +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::evaluation::Evaluation; + use crate::core::objective::{Objective, ObjectiveSpace}; + use crate::operators::real::RealBounds; + + /// A multi-fidelity Sphere1D where higher budgets give a less noisy + /// estimate of `f(x) = x[0]²`. + struct NoisySphere { + noise_decay: f64, // higher noise_decay = less noise per unit budget + } + impl PartialProblem for NoisySphere { + type Decision = Vec; + + fn objectives(&self) -> ObjectiveSpace { + ObjectiveSpace::new(vec![Objective::minimize("f")]) + } + + fn evaluate_at_budget(&self, x: &Vec, budget: f64) -> Evaluation { + // Pure Sphere; the budget controls how much "noise" we add + // (deterministic — no RNG so the test is reproducible). + // Higher budget → smaller residual. + let true_f = x[0] * x[0]; + let residual = (1.0 / (budget * self.noise_decay)).min(10.0); + Evaluation::new(vec![true_f + residual]) + } + } + + #[test] + fn hyperband_finds_minimum() { + let problem = NoisySphere { noise_decay: 1.0 }; + let mut opt = Hyperband::new( + HyperbandConfig { + max_budget: 81.0, + eta: 3.0, + max_brackets: 4, + seed: 1, + }, + RealBounds::new(vec![(-5.0, 5.0)]), + ); + let r = opt.run(&problem); + let best = r.best.unwrap(); + // The "true" minimum of Sphere is 0; but at finite budget the + // residual term keeps it from being zero. A good run should at + // least clearly beat random. + assert!( + best.evaluation.objectives[0] < 0.5, + "got f = {}", + best.evaluation.objectives[0], + ); + assert!(r.evaluations > 0); + } + + #[test] + fn hyperband_deterministic_with_same_seed() { + let make = || { + Hyperband::new( + HyperbandConfig { + max_budget: 27.0, + eta: 3.0, + max_brackets: 3, + seed: 99, + }, + RealBounds::new(vec![(-5.0, 5.0)]), + ) + }; + let problem = NoisySphere { noise_decay: 1.0 }; + let mut a = make(); + let mut b = make(); + let ra = a.run(&problem); + let rb = b.run(&problem); + assert_eq!( + ra.best.unwrap().evaluation.objectives, + rb.best.unwrap().evaluation.objectives, + ); + } + + #[test] + #[should_panic(expected = "exactly one objective")] + fn hyperband_multi_objective_panics() { + struct MultiObj; + impl PartialProblem for MultiObj { + type Decision = Vec; + fn objectives(&self) -> ObjectiveSpace { + ObjectiveSpace::new(vec![ + Objective::minimize("a"), + Objective::minimize("b"), + ]) + } + fn evaluate_at_budget(&self, _: &Vec, _: f64) -> Evaluation { + Evaluation::new(vec![0.0, 0.0]) + } + } + let mut opt = Hyperband::new( + HyperbandConfig::default(), + RealBounds::new(vec![(0.0, 1.0)]), + ); + let _ = opt.run(&MultiObj); + } +} diff --git a/src/algorithms/mod.rs b/src/algorithms/mod.rs index 1ead04c..98dc015 100644 --- a/src/algorithms/mod.rs +++ b/src/algorithms/mod.rs @@ -10,6 +10,7 @@ pub mod genetic_algorithm; pub mod grea; pub mod hill_climber; pub mod hype; +pub mod hyperband; pub mod ibea; pub mod ipop_cma_es; pub mod knea; @@ -44,6 +45,7 @@ pub use genetic_algorithm::*; pub use grea::*; pub use hill_climber::*; pub use hype::*; +pub use hyperband::*; pub use ibea::*; pub use ipop_cma_es::*; pub use knea::*; diff --git a/src/core/mod.rs b/src/core/mod.rs index f8f4f55..54ff67b 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -3,6 +3,7 @@ pub mod candidate; pub mod evaluation; pub mod objective; +pub mod partial_problem; pub mod population; pub mod problem; pub mod result; @@ -11,6 +12,7 @@ pub mod rng; pub use candidate::*; pub use evaluation::*; pub use objective::*; +pub use partial_problem::*; pub use population::*; pub use problem::*; pub use result::*; diff --git a/src/core/partial_problem.rs b/src/core/partial_problem.rs new file mode 100644 index 0000000..95b5531 --- /dev/null +++ b/src/core/partial_problem.rs @@ -0,0 +1,39 @@ +//! Trait for multi-fidelity optimization problems. + +use crate::core::evaluation::Evaluation; +use crate::core::objective::ObjectiveSpace; + +/// A problem whose evaluation cost can be controlled by a fidelity +/// "budget" parameter — for example, an ML training run that gets +/// trained for `budget` epochs, a CFD simulation that runs for `budget` +/// timesteps, or a Monte Carlo evaluation that draws `budget` samples. +/// +/// Multi-fidelity optimizers (Hyperband, Successive Halving, BOHB) use +/// this trait to evaluate cheap low-budget previews of many +/// configurations, then "promote" the survivors to higher budgets. +/// +/// `PartialProblem` is intentionally NOT a sub-trait of [`Problem`] +/// because the evaluation contract is different: `Problem::evaluate` +/// is single-shot, while `evaluate_at_budget` is parameterized by +/// fidelity. Implementors who already have a `Problem` and want their +/// `evaluate_at_budget` to ignore the budget can write a one-line +/// wrapper that just calls `Problem::evaluate`. +/// +/// [`Problem`]: crate::core::Problem +pub trait PartialProblem { + /// The thing the optimizer changes. Same constraints as + /// [`Problem::Decision`](crate::core::Problem::Decision). + type Decision: Clone; + + /// Return the objectives for this problem. + fn objectives(&self) -> ObjectiveSpace; + + /// Evaluate `decision` at the given fidelity `budget`. + /// + /// Higher `budget` should give a more accurate (and more expensive) + /// estimate of the same underlying objective. Hyperband requires + /// monotonicity: a higher-budget evaluation should not be worse + /// than a lower-budget evaluation by chance — though some noise is + /// fine and expected. + fn evaluate_at_budget(&self, decision: &Self::Decision, budget: f64) -> Evaluation; +} diff --git a/src/prelude.rs b/src/prelude.rs index 416c653..7bf05f3 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,7 +6,7 @@ pub use crate::core::{ Candidate, Direction, Evaluation, Objective, ObjectiveSpace, OptimizationResult, - Population, Problem, Rng, rng_from_seed, + PartialProblem, Population, Problem, Rng, rng_from_seed, }; pub use crate::traits::{Initializer, Optimizer, Repair, Variation}; @@ -27,7 +27,8 @@ pub use crate::algorithms::{ BayesianOptConfig, CmaEs, CmaEsConfig, DifferentialEvolution, DifferentialEvolutionConfig, EpsilonMoea, EpsilonMoeaConfig, GeneticAlgorithm, GeneticAlgorithmConfig, Grea, GreaConfig, HillClimber, HillClimberConfig, Hype, - HypeConfig, Ibea, IbeaConfig, IpopCmaEs, IpopCmaEsConfig, Knea, KneaConfig, Moead, MoeadConfig, Mopso, MopsoConfig, + HypeConfig, Hyperband, HyperbandConfig, Ibea, IbeaConfig, IpopCmaEs, IpopCmaEsConfig, + Knea, KneaConfig, Moead, MoeadConfig, Mopso, MopsoConfig, NelderMead, NelderMeadConfig, Nsga2, Nsga2Config, Nsga3, Nsga3Config, OnePlusOneEs, OnePlusOneEsConfig, Paes, PaesConfig, ParticleSwarm, PesaII, PesaIIConfig, ParticleSwarmConfig, RandomSearch, RandomSearchConfig, Rvea, RveaConfig,