diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ab98e..97ad7a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +#### New algorithms (the "expensive-eval and gradient-free" cohort) + +- `OnePlusOneEs` — Rechenberg 1973 (1+1)-ES with the one-fifth success + rule. Smallest possible self-adapting evolution strategy. +- `NelderMead` — Nelder & Mead 1965 simplex direct-search method. Fills + a real gap: heuropt's first classical gradient-free local optimizer. +- `IpopCmaEs` — Auger & Hansen 2005 increasing-population CMA-ES with + restart. Specifically fixes vanilla CMA-ES's known weakness on + multimodal problems (e.g. Rastrigin: 2.35 → 0.13). +- `BayesianOpt` — Gaussian-process-based Bayesian Optimization with + Expected Improvement acquisition. heuropt's first sample-efficient + algorithm: targets the 50–500 evaluation regime where every other + algorithm is way over-budget. + +#### Internal helpers + +- `internal::cholesky` — Cholesky factorization + triangular solves + for symmetric positive-definite matrices, used by the GP posterior + in `BayesianOpt`. Hand-rolled to avoid pulling in nalgebra. + +#### CmaEs API change (additive) + +- `CmaEsConfig` gained an `initial_mean: Option>` field + (defaulting to `None`, which keeps the existing midpoint-of-bounds + behavior). `IpopCmaEs` uses it to inject restart diversity without + shrinking the search box. + ## [0.2.0] — 2026-05-05 A substantial expansion of the algorithm catalog (21 new algorithms), diff --git a/examples/compare.rs b/examples/compare.rs index 044c02f..8b50a12 100644 --- a/examples/compare.rs +++ b/examples/compare.rs @@ -1015,6 +1015,123 @@ fn rastrigin_cma_es(seed: u64) -> SoRun { } } +fn rastrigin_ipop_cma_es(seed: u64) -> SoRun { + let problem = Rastrigin { dim: RASTRIGIN_DIM }; + let bounds = RealBounds::new(vec![(-5.12, 5.12); RASTRIGIN_DIM]); + let pop = 16; + let config = IpopCmaEsConfig { + initial_population_size: pop, + total_generations: RASTRIGIN_BUDGET / pop, + initial_sigma: 1.0, + eigen_decomposition_period: 1, + stall_generations: None, + seed, + }; + let mut opt = IpopCmaEs::new(config, bounds); + let t0 = Instant::now(); + let result = opt.run(&problem); + SoRun { + best_value: result.best.unwrap().evaluation.objectives[0], + wall_ms: t0.elapsed().as_millis(), + } +} + +fn rastrigin_one_plus_one_es(seed: u64) -> SoRun { + let problem = Rastrigin { dim: RASTRIGIN_DIM }; + let bounds = RealBounds::new(vec![(-5.12, 5.12); RASTRIGIN_DIM]); + let config = OnePlusOneEsConfig { + iterations: RASTRIGIN_BUDGET, + initial_sigma: 1.0, + adaptation_period: 50, + step_increase: 1.22, + seed, + }; + let mut opt = OnePlusOneEs::new(config, bounds); + let t0 = Instant::now(); + let result = opt.run(&problem); + SoRun { + best_value: result.best.unwrap().evaluation.objectives[0], + wall_ms: t0.elapsed().as_millis(), + } +} + +fn rosenbrock_nelder_mead(_seed: u64) -> SoRun { + let problem = rosenbrock_problem(); + let bounds = RealBounds::new(vec![(-5.0, 10.0); ROSENBROCK_DIM]); + let config = NelderMeadConfig { + iterations: ROSENBROCK_BUDGET / 4, + ..NelderMeadConfig::default() + }; + let mut opt = NelderMead::new(config, bounds); + let t0 = Instant::now(); + let result = opt.run(&problem); + SoRun { + best_value: result.best.unwrap().evaluation.objectives[0], + wall_ms: t0.elapsed().as_millis(), + } +} + +fn rosenbrock_one_plus_one_es(seed: u64) -> SoRun { + let problem = rosenbrock_problem(); + let bounds = RealBounds::new(vec![(-5.0, 10.0); ROSENBROCK_DIM]); + let config = OnePlusOneEsConfig { + iterations: ROSENBROCK_BUDGET, + initial_sigma: 1.0, + adaptation_period: 30, + step_increase: 1.22, + seed, + }; + let mut opt = OnePlusOneEs::new(config, bounds); + let t0 = Instant::now(); + let result = opt.run(&problem); + SoRun { + best_value: result.best.unwrap().evaluation.objectives[0], + wall_ms: t0.elapsed().as_millis(), + } +} + +fn rosenbrock_bo(seed: u64) -> SoRun { + let problem = rosenbrock_problem(); + let bounds = RealBounds::new(vec![(-5.0, 10.0); ROSENBROCK_DIM]); + let config = BayesianOptConfig { + initial_samples: 10, + iterations: 50, + length_scales: None, + signal_variance: 1.0, + noise_variance: 1e-6, + acquisition_samples: 1_000, + seed, + }; + let mut opt = BayesianOpt::new(config, bounds); + let t0 = Instant::now(); + let result = opt.run(&problem); + SoRun { + best_value: result.best.unwrap().evaluation.objectives[0], + wall_ms: t0.elapsed().as_millis(), + } +} + +fn ackley_bo(seed: u64) -> SoRun { + let problem = ackley_problem(); + let bounds = RealBounds::new(vec![(-32.768, 32.768); ACKLEY_DIM]); + let config = BayesianOptConfig { + initial_samples: 10, + iterations: 50, + length_scales: None, + signal_variance: 1.0, + noise_variance: 1e-6, + acquisition_samples: 1_000, + seed, + }; + let mut opt = BayesianOpt::new(config, bounds); + let t0 = Instant::now(); + let result = opt.run(&problem); + SoRun { + best_value: result.best.unwrap().evaluation.objectives[0], + wall_ms: t0.elapsed().as_millis(), + } +} + // ----------------------------------------------------------------------------- // Rosenbrock + Ackley runners (a curated SO subset on each) // ----------------------------------------------------------------------------- @@ -1440,6 +1557,7 @@ fn run_rastrigin_comparison() { let runners: &[(&str, Runner)] = &[ ("RandomSearch", rastrigin_random), ("HillClimber", rastrigin_hill_climber), + ("(1+1)-ES", rastrigin_one_plus_one_es), ("SimulatedAnneal", rastrigin_simulated_annealing), ("PAES", rastrigin_paes), ("GA", rastrigin_genetic_algorithm), @@ -1447,6 +1565,7 @@ fn run_rastrigin_comparison() { ("NSGA-II", rastrigin_nsga2), ("DE", rastrigin_de), ("CMA-ES", rastrigin_cma_es), + ("IPOP-CMA-ES", rastrigin_ipop_cma_es), ]; for (name, runner) in runners { @@ -1479,6 +1598,9 @@ fn run_rosenbrock_comparison() { ("PSO", rosenbrock_pso), ("CMA-ES", rosenbrock_cma), ("TLBO", rosenbrock_tlbo), + ("(1+1)-ES", rosenbrock_one_plus_one_es), + ("Nelder-Mead", rosenbrock_nelder_mead), + ("BO (60 evals)", rosenbrock_bo), ]; for (name, runner) in runners { let runs: Vec = (0..SEEDS).map(runner).collect(); @@ -1508,6 +1630,7 @@ fn run_ackley_comparison() { ("PSO", ackley_pso), ("CMA-ES", ackley_cma), ("TLBO", ackley_tlbo), + ("BO (60 evals)", ackley_bo), ]; for (name, runner) in runners { let runs: Vec = (0..SEEDS).map(runner).collect();