diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index db2d074fb..172b3736e 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -37,7 +37,6 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests diff --git a/.tools/envs/testenv-nevergrad.yml b/.tools/envs/testenv-nevergrad.yml index 37c74ebd3..8ded70580 100644 --- a/.tools/envs/testenv-nevergrad.yml +++ b/.tools/envs/testenv-nevergrad.yml @@ -34,7 +34,6 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests @@ -42,6 +41,6 @@ dependencies: - types-jinja2 # dev, tests - sqlalchemy-stubs # dev, tests - sphinxcontrib-mermaid # dev, tests, docs + - -e ../../ - bayesian_optimization==1.4.0 - nevergrad - - -e ../../ diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index d5d26c22b..b61ebb87e 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -35,7 +35,6 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 15599c2a3..6a5701e13 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -35,7 +35,6 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index 3aff5dc1d..b766c0512 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -35,7 +35,6 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/.tools/envs/testenv-plotly.yml b/.tools/envs/testenv-plotly.yml index dff80c07e..3dbcbf8f4 100644 --- a/.tools/envs/testenv-plotly.yml +++ b/.tools/envs/testenv-plotly.yml @@ -34,7 +34,6 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests @@ -42,5 +41,5 @@ dependencies: - types-jinja2 # dev, tests - sqlalchemy-stubs # dev, tests - sphinxcontrib-mermaid # dev, tests, docs - - kaleido<0.3 - -e ../../ + - kaleido<0.3 diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md index bdc9c27e5..3b587a6e7 100644 --- a/docs/source/algorithms.md +++ b/docs/source/algorithms.md @@ -3965,573 +3965,378 @@ and hence imprecise.\ `AXP (AX-platfofm)` - Very slow and not recommended. ```{eval-rst} -.. dropdown:: nevergrad_pso +.. dropdown:: nevergrad_pso + + **How to use this algorithm:** .. code-block:: - "nevergrad_pso" - - Minimize a scalar function using the Particle Swarm Optimization algorithm. - - The Particle Swarm Optimization algorithm was originally proposed by :cite:`Kennedy1995`.The - implementation in Nevergrad is based on :cite:`Zambrano2013`. - - PSO solves an optimization problem by evolving a swarm of particles (candidate solutions) across the - search space. Each particle adjusts its position based on its own experience (cognitive component) - and the experiences of its neighbors or the swarm (social component), using velocity updates. The - algorithm iteratively guides the swarm toward promising regions of the search space. - - - **transform** (str): The transform used to map from PSO optimization space to real space. Options: - - "arctan" (default) - - "identity" - - "gaussian" - - **population\_size** (int): The number of particles in the swarm. - - **n\_cores** (int): The number of CPU cores to use for parallel computation. - - **seed** (int, optional): Random seed for reproducibility. - - **stopping\_maxfun** (int, optional): Maximum number of function evaluations. - - **inertia** (float): - Inertia weight ω. Controls the influence of a particle's previous velocity. Must be less than 1 to - avoid divergence. Default is 0.7213475204444817. - - **cognitive** (float): - Cognitive coefficient :math:`\phi_p`. Controls the influence of a particle’s own best known - position. Typical values: 1.0 to 3.0. Default is 1.1931471805599454. - - **social** (float): - Social coefficient. Denoted by :math:`\phi_g`. Controls the influence of the swarm’s best known - position. Typical values: 1.0 to 3.0. Default is 1.1931471805599454. - - **quasi\_opp\_init** (bool): Whether to use quasi-opposition initialization. Default is False. - - **speed\_quasi\_opp\_init** (bool): - Whether to apply quasi-opposition initialization to speed. Default is False. - - **special\_speed\_quasi\_opp\_init** (bool): - Whether to use special quasi-opposition initialization for speed. Default is False. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_pso(stopping_maxfun=1_000, ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_pso", + algo_options={"stopping_maxfun": 1_000, ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradPSO + ``` ```{eval-rst} -.. dropdown:: nevergrad_cmaes +.. dropdown:: nevergrad_cmaes + + **How to use this algorithm:** .. code-block:: - "nevergrad_cmaes" + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_cmaes(stopping_maxfun=1_000, ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_cmaes", + algo_options={"stopping_maxfun": 1_000, ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradCMAES - Minimize a scalar function using the Covariance Matrix Adaptation Evolution Strategy (CMA-ES) - algorithm. - - The CMA-ES (Covariance Matrix Adaptation Evolution Strategy) is a state-of-the-art evolutionary - algorithm designed for difficult non-linear, non-convex, black-box optimization problems in - continuous domains. It is typically applied to unconstrained or bounded optimization problems with - dimensionality between 3 and 100. CMA-ES adapts a multivariate normal distribution to approximate - the shape of the objective function. It estimates a positive-definite covariance matrix, akin to the - inverse Hessian in convex-quadratic problems, but without requiring derivatives or their - approximation. Original paper can be accessed at `cma `_. This - implementation is a python wrapper over the original code `pycma `_. - - - **scale**: Scale of the search. - - **elitist**: - Whether to switch to elitist mode (also known as (μ,λ)-CMA-ES). In elitist mode, the best point in - the population is always retained. - - **population\_size**: Population size. - - **diagonal**: Use the diagonal version of CMA, which is more efficient for high-dimensional problems. - - **high\_speed**: Use a metamodel for recommendation to speed up optimization. - - **fast\_cmaes**: - Use the fast CMA-ES implementation. Cannot be used with diagonal=True. Produces equivalent results - and is preferable for high dimensions or when objective function evaluations are fast. - - **random\_init**: If True, initialize the optimizer with random parameters. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **step\_size\_adaptive**: - Whether to adapt the step size. Can be a boolean or a string specifying the adaptation strategy. - - **CSA\_dampfac**: Damping factor for step size adaptation. - - **CMA\_dampsvec\_fade**: Damping rate for step size adaptation. - - **CSA\_squared**: Whether to use squared step sizes in updates. - - **CMA\_on**: Learning rate for the covariance matrix update. - - **CMA\_rankone**: Multiplier for the rank-one update learning rate of the covariance matrix. - - **CMA\_rankmu**: Multiplier for the rank-mu update learning rate of the covariance matrix. - - **CMA\_cmean**: Learning rate for the mean update. - - **CMA\_diagonal\_decoding**: Learning rate for the diagonal update. - - **num\_parents**: Number of parents (μ) for recombination. - - **CMA\_active**: Whether to use negative updates for the covariance matrix. - - **CMA\_mirrormethod**: Strategy for mirror sampling. Possible values are: - - **0**: Unconditional mirroring - - **1**: Selective mirroring - - **2**: Selective mirroring with delay (default) - - **CMA\_const\_trace**: How to normalize the trace of the covariance matrix. Valid values are: - - False: No normalization - - True: Normalize to 1 - - "arithm": Arithmetic mean normalization - - "geom": Geometric mean normalization - - "aeig": Arithmetic mean of eigenvalues - - "geig": Geometric mean of eigenvalues - - **CMA\_diagonal**: - Number of iterations to use diagonal covariance matrix before switching to full matrix. If False, - always use full matrix. - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **stopping\_maxiter**: Maximum number of iterations before termination. - - **stopping\_timeout**: Maximum time in seconds before termination. - - **stopping\_cov\_mat\_cond**: Maximum condition number of the covariance matrix before termination. - - **convergence\_ftol\_abs**: Absolute tolerance on function value changes for convergence. - - **convergence\_ftol\_rel**: Relative tolerance on function value changes for convergence. - - **convergence\_xtol\_abs**: Absolute tolerance on parameter changes for convergence. - - **convergence\_iter\_noimprove**: Number of iterations without improvement before termination. - - **invariant\_path**: Whether evolution path (pc) should be invariant to transformations. - - **eval\_final\_mean**: Whether to evaluate the final mean solution. - - **seed**: Seed used by the internal random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. ``` ```{eval-rst} .. dropdown:: nevergrad_oneplusone + **How to use this algorithm:** + .. code-block:: - "nevergrad_oneplusone" - - Minimize a scalar function using the One Plus One Evolutionary algorithm from Nevergrad. - - THe One Plus One evolutionary algorithm iterates to find a set of parameters that minimizes the loss - function. It does this by perturbing, or mutating, the parameters from the last iteration (the - parent). If the new (child) parameters yield a better result, then the child becomes the new parent - whose parameters are perturbed, perhaps more aggressively. If the parent yields a better result, it - remains the parent and the next perturbation is less aggressive. Originally proposed by - :cite:`Rechenberg1973`. The implementation in Nevergrad is based on the one-fifth adaptation rule, - going back to :cite:`Schumer1968. - - - **noise\_handling**: Method for handling the noise, can be - - "random": A random point is reevaluated regularly using the one-fifth adaptation rule. - - "optimistic": The best optimistic point is reevaluated regularly, embracing optimism in the face of uncertainty. - - A float coefficient can be provided to tune the regularity of these reevaluations (default is 0.05). Eg: with 0.05, each evaluation has a 5% chance (i.e., 1 in 20) of being repeated (i.e., the same candidate solution is reevaluated to better estimate its performance). (Default: `None`). - - **n\_cores**: Number of cores to use. - - stopping.maxfun: Maximum number of function evaluations. - - **mutation**: Type of mutation to apply. Available options are (Default: `"gaussian"`). - - "gaussian": Standard mutation by adding a Gaussian random variable (with progressive widening) to the best pessimistic point. - - "cauchy": Same as Gaussian but using a Cauchy distribution. - - "discrete": Mutates a randomly drawn variable (mutation occurs with probability 1/d in d dimensions, hence ~1 variable per mutation). - - "discreteBSO": Follows brainstorm optimization by gradually decreasing mutation rate from 1 to 1/d. - - "fastga": Fast Genetic Algorithm mutations from the current best. - - "doublefastga": Double-FastGA mutations from the current best :cite:`doerr2017`. - - "rls": Randomized Local Search — mutates one and only one variable. - - "portfolio": Random number of mutated bits, known as uniform mixing :cite:`dang2016`. - - "lengler": Mutation rate is a function of dimension and iteration index. - - "lengler{2|3|half|fourth}": Variants of the Lengler mutation rate adaptation. - - **sparse**: Whether to apply random mutations that set variables to zero. Default is `False`. - - **smoother**: Whether to suggest smooth mutations. Default is `False`. - - **annealing**: - Annealing schedule to apply to mutation amplitude or temperature-based control. Options are: - - "none": No annealing is applied. - - "Exp0.9": Exponential decay with rate 0.9. - - "Exp0.99": Exponential decay with rate 0.99. - - "Exp0.9Auto": Exponential decay with rate 0.9, auto-scaled based on problem horizon. - - "Lin100.0": Linear decay from 1 to 0 over 100 iterations. - - "Lin1.0": Linear decay from 1 to 0 over 1 iteration. - - "LinAuto": Linearly decaying annealing automatically scaled to the problem horizon. Default is `"none"`. - - **super\_radii**: - Whether to apply extended radii beyond standard bounds for candidate generation, enabling broader - exploration. Default is `False`. - - **roulette\_size**: - Size of the roulette wheel used for selection in the evolutionary process. Affects the sampling - diversity from past candidates. (Default: `64`) - - **antismooth**: - Degree of anti-smoothing applied to prevent premature convergence in smooth landscapes. This alters - the landscape by penalizing overly smooth improvements. (Default: `4`) - - **crossover**: Whether to include a genetic crossover step every other iteration. Default is `False`. - - **crossover\_type**: - Method used for genetic crossover between individuals in the population. Available options (Default: `"none"`): - - "none": No crossover is applied. - - "rand": Randomized selection of crossover point. - - "max": Crossover at the point with maximum fitness gain. - - "min": Crossover at the point with minimum fitness gain. - - "onepoint": One-point crossover, splitting the genome at a single random point. - - "twopoint": Two-point crossover, splitting the genome at two points and exchanging the middle section. - - **tabu\_length**: - Length of the tabu list used to prevent revisiting recently evaluated candidates in local search - strategies. Helps in escaping local minima. (Default: `1000`) - - **rotation**: - Whether to apply rotational transformations to the search space, promoting invariance to axis- - aligned structures and enhancing search performance in rotated coordinate systems. (Default: - `False`) - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_oneplusone(stopping_maxfun=1_000, ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_oneplusone", + algo_options={"stopping_maxfun": 1_000, ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradOnePlusOne ``` ```{eval-rst} .. dropdown:: nevergrad_de + **How to use this algorithm:** + .. code-block:: - "nevergrad_de" - - Minimize a scalar function using the Differential Evolution optimizer from Nevergrad. - - Differential Evolution is typically used for continuous optimization. It uses differences between - points in the population for performing mutations in fruitful directions; it is therefore a kind of - covariance adaptation without any explicit covariance, making it very fast in high dimensions. - - - **initialization**: - Algorithm/distribution used for initialization. Can be one of: "parametrization" (uses - parametrization's sample method), "LHS" (Latin Hypercube Sampling), "QR" (Quasi-Random), "QO" - (Quasi-Orthogonal), or "SO" (Sobol sequence). - - **scale**: Scale of random component of updates. Can be a float or a string. - - **recommendation**: Criterion for selecting the best point to recommend. - - **Options**: "pessimistic", "optimistic", "mean", or "noisy". - - **crossover**: Crossover rate or strategy. Can be: - - float: Fixed crossover rate - - "dimension": 1/dimension - - "random": Random uniform rate per iteration - - "onepoint": One-point crossover - - "twopoints": Two-points crossover - - "rotated_twopoints": Rotated two-points crossover - - "parametrization": Use parametrization's recombine method - - **F1**: Differential weight #1 (scaling factor). - - **F2**: Differential weight #2 (scaling factor). - - **popsize**: Population size. Can be an integer or one of: - - "standard": max(num_workers, 30) - - "dimension": max(num_workers, 30, dimension + 1) - - "large": max(num_workers, 30, 7 * dimension) - - **high\_speed**: If True, uses a metamodel for recommendations to speed up optimization. - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_de(population_size="large", ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_de", + algo_options={"population_size": "large", ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradDifferentialEvolution ``` ```{eval-rst} -.. dropdown:: nevergrad_bo +.. dropdown:: nevergrad_bo + + .. note:: + + Using this optimizer requires the `bayes-optim` package to be installed as well. + This can be done with `pip install bayes-optim`. + + **How to use this algorithm:** + + .. code-block:: + + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_bo(stopping_maxfun=1_000, ...) + ) + + or .. code-block:: - "nevergrad_bo" - - Minimize a scalar function using the Bayes Optim algorithm. BO and PCA-BO algorithms from the - `bayes_optim `_ package PCA-BO (Principal - Component Analysis for Bayesian Optimization) is a dimensionality reduction technique for black-box - optimization. It applies PCA to the input space before performing Bayesian optimization, improving - efficiency in high dimensions by focusing on directions of greatest variance. This helps concentrate - search in informative subspaces and reduce sample complexity. :cite:`bayesoptimimpl`. - - - **init\_budget**: Number of initialization algorithm steps. - - **pca**: Whether to use the PCA transformation, defining PCA-BO rather than standard BO. - - **n\_components**: - Number of principal axes in feature space representing directions of maximum variance in the data. - Represents the percentage of explained variance (e.g., 0.95 means 95% variance retained). - - **prop\_doe\_factor**: - Percentage of the initial budget used for DoE, potentially overriding `init_budget`. For - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + om.minimize( + ..., + algorithm="nevergrad_bo", + algo_options={"stopping_maxfun": 1_000, ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradBayesOptim ``` ```{eval-rst} .. dropdown:: nevergrad_emna + **How to use this algorithm:** + .. code-block:: - "nevergrad_emna" - - Minimize a scalar function using the Estimation of Multivariate Normal Algorithm. - - Estimation of Multivariate Normal Algorithm (EMNA), a distribution-based evolutionary algorithm that - models the search space using a multivariate Gaussian. EMNA learns the full covariance matrix of the - Gaussian sampling distribution, resulting in a cubic time complexity w.r.t. each sampling. It is - highly recommended to first attempt other more advanced optimization methods for LBO. See - :cite:`emnaimpl`. This algorithm is quite efficient in a parallel setting, i.e. when the population - size is large. - - - **isotropic**: - If True, uses an isotropic (identity covariance) Gaussian. If False, uses a separable (diagonal - covariance) Gaussian for greater flexibility in anisotropic landscapes. - - **noise\_handling**: - If True, returns the best individual found. If False (recommended for noisy problems), returns the - average of the final population to reduce noise. - - **population\_size\_adaptation**: - If True, the population size is adjusted automatically based on the optimization landscape and noise - level. - - **initial\_popsize**: Initial population size. Default: 4 x dimension.. - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_emna(noise_handling=False, ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_emna", + algo_options={"noise_handling": False, ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradEMNA ``` ```{eval-rst} .. dropdown:: nevergrad_cga + **How to use this algorithm:** + + .. code-block:: + + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_cga(stopping_maxfun=10_000) + ) + + or + .. code-block:: - "nevergrad_cga" - - Minimize a scalar function using the Compact Genetic Algorithm. - - The Compact Genetic Algorithm (cGA) is a memory-efficient genetic algorithm that represents the - population as a probability vector over gene values. It simulates the order-one behavior of a simple - GA with uniform crossover, updating probabilities instead of maintaining an explicit population. cGA - processes each gene independently and is well-suited for large or constrained environments. For - details see :cite:`cgaimpl`. - - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + om.minimize( + ..., + algorithm="nevergrad_cga", + algo_options={"stopping_maxfun": 10_000} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradCGA ``` ```{eval-rst} .. dropdown:: nevergrad_eda + **How to use this algorithm:** + + .. code-block:: + + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_eda(stopping_maxfun=10_000) + ) + + or + .. code-block:: - "nevergrad_eda" - - Minimize a scalar function using the Estimation of distribution algorithm. - - Estimation of Distribution Algorithms (EDAs) optimize by building and sampling a probabilistic model - of promising solutions. Instead of using traditional variation operators like crossover or mutation, - EDAs update a distribution based on selected individuals and sample new candidates from it. This - allows efficient exploration of complex or noisy search spaces. In short, EDAs typically do not - directly evolve populations of search points but build probabilistic models of promising solutions - by repeatedly sampling and selecting points from the underlying search space. Refer :cite:`edaimpl`. - - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + om.minimize( + ..., + algorithm="nevergrad_eda", + algo_options={"stopping_maxfun": 10_000} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradEDA ``` ```{eval-rst} .. dropdown:: nevergrad_tbpsa + **How to use this algorithm:** + .. code-block:: - "nevergrad_tbpsa" - - Minimize a scalar function using the Test-based population size adaptation algorithm. - - TBPSA adapts population size based on fitness trend detection using linear regression. If no - significant improvement is found (via hypothesis testing), the population size is increased to - improve robustness in noisy settings. This method performs the best in many noisy optimization - problems, even in large dimensions. For more details, refer :cite:`tbpsaimpl` - - - **noise\_handling**: - If True, returns the best individual seen so far. If False (recommended for noisy problems), returns - the average of the final population to reduce the effect of noise. - - **initial\_popsize**: Initial population size. If not specified, defaults to 4 x dimension. - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_tbpsa(noise_handling=False, ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_tbpsa", + algo_options={"noise_handling": False, ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradTBPSA ``` ```{eval-rst} .. dropdown:: nevergrad_randomsearch + **How to use this algorithm:** + .. code-block:: - "nevergrad_randomsearch" - - Minimize a scalar function using the Random Search algorithm. - - This is a one-shot optimization method, provides random suggestions. - - - **middle\_point**: - Enforces that the first suggested point (ask) is the zero vector. i.e we add (0,0,...,0) as a first - point. - - **opposition\_mode**: Symmetrizes exploration with respect to the center. - - "opposite": enables full symmetry by always evaluating mirrored points. - - "quasi": applies randomized symmetry (less strict, more exploratory). - - None: disables any symmetric mirroring in the sampling process. - - **sampler**: - - "parametrization": uses the default sample() method of the parametrization, which samples uniformly within bounds or from a Gaussian. - - "gaussian": samples from a standard Gaussian distribution. - - "cauchy": uses a Cauchy distribution instead of Gaussian. - - **scale**: Scalar used to multiply suggested point values, or a string mode: - - "random": uses a randomized pattern for the scale. - - "auto": sigma = (1 + log(budget)) / (4 * log(dimension)); adjusts scale based on problem size. - - "autotune": sigma = sqrt(log(budget) / dimension); alternative auto-scaling based on budget and dimensionality. - - **recommendation\_rule**: Specifies how the final recommendation is chosen. - - "average_of_best": returns the average of top-performing candidates. - - "pessimistic": selects the pessimistic best (default); - - "average_of_exp_best": uses an exponential moving average of the best points. - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_randomsearch(opposition_mode="quasi", ...) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_randomsearch", + algo_options={"opposition_mode": "quasi", ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradRandomSearch ``` ```{eval-rst} .. dropdown:: nevergrad_samplingsearch + **How to use this algorithm:** + .. code-block:: - "nevergrad_samplingsearch" - - Minimize a scalar function using SamplingSearch. - - This is a one-shot optimization method, but better than random search by ensuring more uniformity. - - - **sampler**: Choice of the low-discrepancy sampler used for initial points. - - "Halton": deterministic, well-spaced sequences - - "Hammersley": similar to Halton but more uniform in low dimension - - "LHS": Latin Hypercube Sampling; ensures coverage along each axis - - **scrambled**: - If True, Adds scrambling to the search; much better in high dimension and rarely worse than the - original search. - - **middle\_point**: - If True, the first suggested point is the zero vector. Useful for initializing at the center of the - search space. - - **cauchy**: - If True, uses the inverse Cauchy distribution instead of Gaussian when projecting samples to real- - valued space (especially when no box bounds exist). - - **scale**: A float multiplier or "random". - - float: directly scales all generated points - - "random": uses a randomized scaling pattern for increased diversity - - **rescaled**: If True or a specific mode, rescales the sampling pattern. - - Ensures coverage of boundaries and may apply adaptive scaling - - Useful when original scale is too narrow or biased - - **recommendation\_rule**: How the final recommendation is chosen. - - "average_of_best": mean of the best-performing points - - "pessimistic": selects the point with best worst-case value (default) - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. Notes - ----- - - Halton is a low quality sampling method when the dimension is high; it is usually better to use Halton with scrambling. - - When the budget is known in advance, it is also better to replace Halton by Hammersley. + import optimagic as om + om.minimize( + ..., + algorithm=om.algos.nevergrad_samplingsearch(sampler="Hammersley", scrambled=True) + ) + + or + + .. code-block:: + + om.minimize( + ..., + algorithm="nevergrad_samplingsearch", + algo_options={"sampler": "Hammersley", "scrambled": True} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradSamplingSearch ``` ```{eval-rst} -.. dropdown:: nevergrad_NGOpt +.. dropdown:: nevergrad_wizard + + **How to use this algorithm:** + + .. code-block:: + + import optimagic as om + from optimagic.optimizers.nevergrad_optimizers import Wizard + om.minimize( + ..., + algorithm=om.algos.nevergrad_wizard(optimizer= Wizard.NGOptRW, ...) + ) + + or .. code-block:: - "nevergrad_NGOpt" - - Minimize a scalar function using a Meta Optimizer from Nevergrad. Each meta optimizer combines - multiples optimizers to solve a problem. - - - **optimizer**: One of - - NGOpt - - NGOpt4 - - NGOpt8 - - NGOpt10 - - NGOpt12 - - NGOpt13 - - NGOpt14 - - NGOpt15 - - NGOpt16 - - NGOpt21 - - NGOpt36 - - NGOpt38 - - NGOpt39 - - NGOptRW - - NGOptF - - NGOptF2 - - NGOptF3 - - NGOptF5 - - NgIoh2 - - NgIoh3 - - NgIoh4 - - NgIoh5 - - NgIoh6 - - NgIoh7 - - NgIoh8 - - NgIoh9 - - NgIoh10 - - NgIoh11 - - NgIoh12 - - NgIoh13 - - NgIoh14 - - NgIoh15 - - NgIoh16 - - NgIoh17 - - NgIoh18 - - NgIoh19 - - NgIoh20 - - NgIoh21 - - NgIoh12b - - NgIoh13b - - NgIoh14b - - NgIoh15b - - NgIohRW2 - - NgIohTuned - - NgDS - - NgDS2 - - NGDSRW - - NGO - - CSEC - - CSEC10 - - CSEC11 - - Wiz - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + om.minimize( + ..., + algorithm="nevergrad_wizard", + algo_options={"optimizer": "NGOptRW", ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradWizard + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.Wizard ``` ```{eval-rst} -.. dropdown:: nevergrad_meta +.. dropdown:: nevergrad_portfolio + + **How to use this algorithm:** + + .. code-block:: + + import optimagic as om + from optimagic.optimizers.nevergrad_optimizers import Portfolio + om.minimize( + ..., + algorithm=om.algos.nevergrad_portfolio(optimizer= Portfolio.BFGSCMAPlus, ...) + ) + + or .. code-block:: - "nevergrad_meta" - - Minimize a scalar function using a Meta Optimizer from Nevergrad. Utilizes a combination of local - and global optimizers to find the best solution. Local optimizers like BFGS are wrappers over scipy - implementations. Each meta optimizer combines multiples optimizers to solve a problem. - - - **optimizer**: One of - - MultiBFGSPlus - - LogMultiBFGSPlus - - SqrtMultiBFGSPlus - - MultiCobylaPlus - - MultiSQPPlus - - BFGSCMAPlus - - LogBFGSCMAPlus - - SqrtBFGSCMAPlus - - SQPCMAPlus - - LogSQPCMAPlus - - SqrtSQPCMAPlus - - MultiBFGS - - LogMultiBFGS - - SqrtMultiBFGS - - MultiCobyla - - ForceMultiCobyla - - MultiSQP - - BFGSCMA - - LogBFGSCMA - - SqrtBFGSCMA - - SQPCMA - - LogSQPCMA - - SqrtSQPCMA - - FSQPCMA - - F2SQPCMA - - F3SQPCMA - - MultiDiscrete - - CMandAS2 - - CMandAS3 - - MetaCMA - - CMA - - PCEDA - - MPCEDA - - MEDA - - NoisyBandit - - Shiwa - - Carola3 - - **stopping\_maxfun**: Maximum number of function evaluations before termination. - - **n\_cores**: Number of cores to use for parallel function evaluation. - - **seed**: Seed for the random number generator for reproducibility. - - **sigma**: - Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. + om.minimize( + ..., + algorithm="nevergrad_portfolio", + algo_options={"optimizer": "BFGSCMAPlus", ...} + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.NevergradPortfolio + .. autoclass:: optimagic.optimizers.nevergrad_optimizers.Portfolio ``` ## Bayesian Optimization @@ -4540,6 +4345,8 @@ We wrap the [BayesianOptimization](https://github.com/bayesian-optimization/BayesianOptimization) package. To use it, you need to have [bayesian-optimization](https://pypi.org/project/bayesian-optimization/) installed. +Note: This optimizer requires `bayesian_optimization > 2.0.0` to be installed which is +incompatible with `nevergrad > 1.0.3`. ```{eval-rst} .. dropdown:: bayes_opt @@ -4570,81 +4377,6 @@ package. To use it, you need to have ``` -```{eval-rst} -.. dropdown:: nevergrad_oneplusone - - .. code-block:: - - "nevergrad_oneplusone" - - Minimize a scalar function using the One Plus One Evolutionary algorithm from Nevergrad. - - THe One Plus One evolutionary algorithm iterates to find a set of parameters that minimizes the loss - function. It does this by perturbing, or mutating, the parameters from the last iteration (the - parent). If the new (child) parameters yield a better result, then the child becomes the new parent - whose parameters are perturbed, perhaps more aggressively. If the parent yields a better result, it - remains the parent and the next perturbation is less aggressive. Originally proposed by - :cite:`Rechenberg1973`. The implementation in Nevergrad is based on the one-fifth adaptation rule, - going back to :cite:`Schumer1968. - - - **noise\_handling**: Method for handling the noise, can be - - "random": A random point is reevaluated regularly using the one-fifth adaptation rule. - - "optimistic": The best optimistic point is reevaluated regularly, embracing optimism in the face of uncertainty. - - A float coefficient can be provided to tune the regularity of these reevaluations (default is 0.05). Eg: with 0.05, each evaluation has a 5% chance (i.e., 1 in 20) of being repeated (i.e., the same candidate solution is reevaluated to better estimate its performance). (Default: `None`). - - **n\_cores**: Number of cores to use. - - - **stopping.maxfun**: Maximum number of function evaluations. - - **mutation**: Type of mutation to apply. Available options are (Default: `"gaussian"`). - - "gaussian": Standard mutation by adding a Gaussian random variable (with progressive widening) to the best pessimistic point. - - "cauchy": Same as Gaussian but using a Cauchy distribution. - - "discrete": Mutates a randomly drawn variable (mutation occurs with probability 1/d in d dimensions, hence ~1 variable per mutation). - - "discreteBSO": Follows brainstorm optimization by gradually decreasing mutation rate from 1 to 1/d. - - "fastga": Fast Genetic Algorithm mutations from the current best. - - "doublefastga": Double-FastGA mutations from the current best :cite:`doerr2017`. - - "rls": Randomized Local Search — mutates one and only one variable. - - "portfolio": Random number of mutated bits, known as uniform mixing :cite:`dang2016`. - - "lengler": Mutation rate is a function of dimension and iteration index. - - "lengler{2|3|half|fourth}": Variants of the Lengler mutation rate adaptation. - - **sparse**: Whether to apply random mutations that set variables to zero. Default is `False`. - - **smoother**: Whether to suggest smooth mutations. Default is `False`. - - **annealing**: - Annealing schedule to apply to mutation amplitude or temperature-based control. Options are: - - "none": No annealing is applied. - - "Exp0.9": Exponential decay with rate 0.9. - - "Exp0.99": Exponential decay with rate 0.99. - - "Exp0.9Auto": Exponential decay with rate 0.9, auto-scaled based on problem horizon. - - "Lin100.0": Linear decay from 1 to 0 over 100 iterations. - - "Lin1.0": Linear decay from 1 to 0 over 1 iteration. - - "LinAuto": Linearly decaying annealing automatically scaled to the problem horizon. Default is `"none"`. - - **super\_radii**: - Whether to apply extended radii beyond standard bounds for candidate generation, enabling broader - exploration. Default is `False`. - - **roulette\_size**: - Size of the roulette wheel used for selection in the evolutionary process. Affects the sampling - diversity from past candidates. (Default: `64`) - - **antismooth**: - Degree of anti-smoothing applied to prevent premature convergence in smooth landscapes. This alters - the landscape by penalizing overly smooth improvements. (Default: `4`) - - **crossover**: Whether to include a genetic crossover step every other iteration. Default is `False`. - - **crossover\_type**: - Method used for genetic crossover between individuals in the population. Available options (Default: `"none"`): - - "none": No crossover is applied. - - "rand": Randomized selection of crossover point. - - "max": Crossover at the point with maximum fitness gain. - - "min": Crossover at the point with minimum fitness gain. - - "onepoint": One-point crossover, splitting the genome at a single random point. - - "twopoint": Two-point crossover, splitting the genome at two points and exchanging the middle section. - - **tabu\_length**: - Length of the tabu list used to prevent revisiting recently evaluated candidates in local search - strategies. Helps in escaping local minima. (Default: `1000`) - - **rotation**: - Whether to apply rotational transformations to the search space, promoting invariance to axis- - aligned structures and enhancing search performance in rotated coordinate systems. (Default: - `False`) - - **seed**: Seed for the random number generator for reproducibility. - -``` - ## Gradient Free Optimizers Optimizers from the diff --git a/docs/source/how_to/how_to_start_parameters.md b/docs/source/how_to/how_to_start_parameters.md index fc5a031e9..0c13ba6bf 100644 --- a/docs/source/how_to/how_to_start_parameters.md +++ b/docs/source/how_to/how_to_start_parameters.md @@ -14,125 +14,120 @@ advantages and drawbacks of each of them. Again, we use the simple `sphere` function you know from other tutorials as an example. ```{eval-rst} -.. tabbed:: Array - A frequent choice of ``params`` is a one-dimensional numpy array. This is - because one-dimensional numpy arrays are all that is supported by most optimizer - libraries. +.. tab-set:: + .. tab-item:: Array - In our opinion, it is rarely a good choice to represent parameters as flat numpy arrays - and then access individual parameters or sclices by positions. The only exception - are simple optimization problems with very-fast-to-evaluate criterion functions where - any overhead must be avoided. + A frequent choice of ``params`` is a one-dimensional numpy array. This is + because one-dimensional numpy arrays are all that is supported by most optimizer + libraries. - If you still want to use one-dimensional numpy arrays, here is how: + In our opinion, it is rarely a good choice to represent parameters as flat numpy arrays + and then access individual parameters or sclices by positions. The only exception + are simple optimization problems with very-fast-to-evaluate criterion functions where + any overhead must be avoided. - .. code-block:: python + If you still want to use one-dimensional numpy arrays, here is how: - import optimagic as om + .. code-block:: python + import optimagic as om - def sphere(params): - return params @ params + def sphere(params): + return params @ params - om.minimize( - fun=sphere, - params=np.arange(3), - algorithm="scipy_lbfgsb", - ) -``` + om.minimize( + fun=sphere, + params=np.arange(3), + algorithm="scipy_lbfgsb", + ) -```{eval-rst} -.. tabbed:: DataFrame + .. tab-item:: DataFrame - Originally, pandas DataFrames were the mandatory format for ``params`` in optimagic. - They are still highly recommended and have a few special features. For example, - they allow to bundle information on start parameters and bounds together into one - data structure. + Originally, pandas DataFrames were the mandatory format for ``params`` in optimagic. + They are still highly recommended and have a few special features. For example, + they allow to bundle information on start parameters and bounds together into one + data structure. - Let's look at an example where we do that: + Let's look at an example where we do that: - .. code-block:: python + .. code-block:: python - def sphere(params): - return (params["value"] ** 2).sum() + def sphere(params): + return (params["value"] ** 2).sum() - params = pd.DataFrame( - data={"value": [1, 2, 3], "lower_bound": [-np.inf, 1.5, 0]}, - index=["a", "b", "c"], - ) + params = pd.DataFrame( + data={"value": [1, 2, 3], "lower_bound": [-np.inf, 1.5, 0]}, + index=["a", "b", "c"], + ) - om.minimize( - fun=sphere, - params=params, - algorithm="scipy_lbfgsb", - ) + om.minimize( + fun=sphere, + params=params, + algorithm="scipy_lbfgsb", + ) - DataFrames have many advantages: + DataFrames have many advantages: - - It is easy to select single parameters or groups of parameters or work with - the entire parameter vector. Especially, if you use a well designed MultiIndex. - - It is very easy to produce publication quality LaTeX tables from them. - - If you have nested models, you can easily update the parameter vector of a larger - model with the values from a smaller one (e.g. to get good start parameters). - - You can bundle information on bounds and values in one place. - - It is easy to compare two params vectors for equality. + - It is easy to select single parameters or groups of parameters or work with + the entire parameter vector. Especially, if you use a well designed MultiIndex. + - It is very easy to produce publication quality LaTeX tables from them. + - If you have nested models, you can easily update the parameter vector of a larger + model with the values from a smaller one (e.g. to get good start parameters). + - You can bundle information on bounds and values in one place. + - It is easy to compare two params vectors for equality. - If you are sure you won't have bounds on your parameter, you can also use a - pandas.Series instead of a pandas.DataFrame. + If you are sure you won't have bounds on your parameter, you can also use a + pandas.Series instead of a pandas.DataFrame. - A drawback of DataFrames is that they are not JAX compatible. Another one is that - they are a bit slower than numpy arrays. + A drawback of DataFrames is that they are not JAX compatible. Another one is that + they are a bit slower than numpy arrays. -``` + .. tab-item:: Dict -```{eval-rst} -.. tabbed:: Dict + ``params`` can also be a (nested) dictionary containing all of the above and more. - ``params`` can also be a (nested) dictionary containing all of the above and more. + .. code-block:: python - .. code-block:: python + def sphere(params): + return params["a"] ** 2 + params["b"] ** 2 + (params["c"] ** 2).sum() - def sphere(params): - return params["a"] ** 2 + params["b"] ** 2 + (params["c"] ** 2).sum() + res = om.minimize( + fun=sphere, + params={"a": 0, "b": 1, "c": pd.Series([2, 3, 4])}, + algorithm="scipy_neldermead", + ) - res = om.minimize( - fun=sphere, - params={"a": 0, "b": 1, "c": pd.Series([2, 3, 4])}, - algorithm="scipy_neldermead", - ) + Dictionarys of arrays are ideal if you want to do vectorized computations with + groups of parameters. They are also a good choice if you calculate derivatives + with JAX. - Dictionarys of arrays are ideal if you want to do vectorized computations with - groups of parameters. They are also a good choice if you calculate derivatives - with JAX. + While optimagic won't stop you, don't go too far! Having parameters in very deeply + nested dictionaries makes it hard to visualize results and/or even to compare two + estimation results. - While optimagic won't stop you, don't go too far! Having parameters in very deeply - nested dictionaries makes it hard to visualize results and/or even to compare two - estimation results. -``` + .. tab-item:: Scalar -```{eval-rst} -.. tabbed:: Scalar + If you have a one-dimensional optimization problem, the natural way to represent + your params is a float: - If you have a one-dimensional optimization problem, the natural way to represent - your params is a float: + .. code-block:: python - .. code-block:: python + def sphere(params): + return params**2 - def sphere(params): - return params**2 + om.minimize( + fun=sphere, + params=3, + algorithm="scipy_lbfgsb", + ) - om.minimize( - fun=sphere, - params=3, - algorithm="scipy_lbfgsb", - ) ``` diff --git a/docs/source/refs.bib b/docs/source/refs.bib index 485894194..89ab3790e 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -964,8 +964,8 @@ @inproceedings{tbpsaimpl year = {2016}, month = {09}, pages = {}, -title = {Evolution under Strong Noise: A Self-Adaptive Evolution Strategy Can Reach the Lower Performance Bound - the pcCMSA-ES}, -volume = {9921}, +title = {Evolution under Strong Noise: A Self-Adaptive Evolution Strategy Can Reach the Lower Performance Bound - the pcCMSA-ES}, +booktitle = {Parallel Problem Solving from Nature -- PPSN XIII},volume = {9921}, isbn = {9783319458229}, doi = {10.1007/978-3-319-45823-6_3} } @@ -1037,6 +1037,7 @@ @book{emnaimpl pages = {}, title = {Estimation of Distribution Algorithms: A New Tool for Evolutionary Computation}, isbn = {9781461356042}, +publisher = {Springer}, journal = {Genetic algorithms and evolutionary computation ; 2}, doi = {10.1007/978-1-4615-1539-5} } diff --git a/environment.yml b/environment.yml index 27a69d1e5..1524ad677 100644 --- a/environment.yml +++ b/environment.yml @@ -49,7 +49,6 @@ dependencies: - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - pre-commit>=4 # dev - - bayes_optim # dev, tests - gradient_free_optimizers # dev, tests - -e . # dev # type stubs diff --git a/pyproject.toml b/pyproject.toml index ac9ad7ded..455da3870 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,6 +383,7 @@ module = [ "pdbp", "iminuit", "nevergrad", + "nevergrad.optimization.base", "pygad", "yaml", "gradient_free_optimizers", diff --git a/src/optimagic/algorithms.py b/src/optimagic/algorithms.py index c7fa34cb2..a7aa199f3 100644 --- a/src/optimagic/algorithms.py +++ b/src/optimagic/algorithms.py @@ -34,13 +34,13 @@ NevergradDifferentialEvolution, NevergradEDA, NevergradEMNA, - NevergradMeta, - NevergradNGOpt, NevergradOnePlusOne, + NevergradPortfolio, NevergradPSO, NevergradRandomSearch, NevergradSamplingSearch, NevergradTBPSA, + NevergradWizard, ) from optimagic.optimizers.nlopt_optimizers import ( NloptBOBYQA, @@ -202,13 +202,13 @@ class BoundedGlobalGradientFreeParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -418,13 +418,13 @@ class BoundedGlobalGradientFreeScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -471,13 +471,13 @@ class BoundedGlobalGradientFreeParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -540,13 +540,13 @@ class GlobalGradientFreeParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -701,13 +701,13 @@ class BoundedGradientFreeParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -809,13 +809,13 @@ class BoundedGlobalParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -1158,13 +1158,13 @@ class BoundedGlobalGradientFreeAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -1244,13 +1244,13 @@ class GlobalGradientFreeScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -1301,13 +1301,13 @@ class GlobalGradientFreeParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -1489,13 +1489,13 @@ class BoundedGradientFreeScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -1574,13 +1574,13 @@ class BoundedGradientFreeParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pounders: Type[Pounders] = Pounders pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco @@ -1669,13 +1669,13 @@ class GradientFreeParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -1760,13 +1760,13 @@ class BoundedGlobalScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -1822,13 +1822,13 @@ class BoundedGlobalParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -1904,13 +1904,13 @@ class GlobalParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -2152,13 +2152,13 @@ class BoundedParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -2433,13 +2433,13 @@ class GlobalGradientFreeAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -2543,13 +2543,13 @@ class BoundedGradientFreeAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -2662,13 +2662,13 @@ class GradientFreeScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -2756,13 +2756,13 @@ class GradientFreeParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pounders: Type[Pounders] = Pounders pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco @@ -2818,13 +2818,13 @@ class BoundedGlobalAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -2922,13 +2922,13 @@ class GlobalScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -2988,13 +2988,13 @@ class GlobalParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -3275,13 +3275,13 @@ class BoundedScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -3389,13 +3389,13 @@ class BoundedParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pounders: Type[Pounders] = Pounders pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco @@ -3504,13 +3504,13 @@ class ParallelScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -3634,13 +3634,13 @@ class GradientFreeAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -3727,13 +3727,13 @@ class GlobalAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -3883,13 +3883,13 @@ class BoundedAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -4042,13 +4042,13 @@ class ScalarAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -4184,13 +4184,13 @@ class ParallelAlgorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard pounders: Type[Pounders] = Pounders pygad: Type[Pygad] = Pygad pygmo_gaco: Type[PygmoGaco] = PygmoGaco @@ -4255,13 +4255,13 @@ class Algorithms(AlgoSelection): nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution nevergrad_eda: Type[NevergradEDA] = NevergradEDA nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA - nevergrad_meta: Type[NevergradMeta] = NevergradMeta - nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_portfolio: Type[NevergradPortfolio] = NevergradPortfolio nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA + nevergrad_wizard: Type[NevergradWizard] = NevergradWizard nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA diff --git a/src/optimagic/config.py b/src/optimagic/config.py index 5d8563502..039d25149 100644 --- a/src/optimagic/config.py +++ b/src/optimagic/config.py @@ -52,7 +52,17 @@ def _is_installed(module_name: str) -> bool: IS_NUMBA_INSTALLED = _is_installed("numba") IS_IMINUIT_INSTALLED = _is_installed("iminuit") IS_NEVERGRAD_INSTALLED = _is_installed("nevergrad") -IS_BAYESOPT_INSTALLED = _is_installed("bayes_opt") +# despite the similar names, the bayes_opt and bayes_optim packages are +# completely unrelated. However, both of them are dependencies of nevergrad. +IS_BAYESOPTIM_INSTALLED = _is_installed("bayes-optim") +# Note: There is a dependancy conflict with nevergrad and bayesian_optimization +# installing nevergrad pins bayesian_optimization to 1.4.0, +# but "bayes_opt" requires bayesian_optimization>=2.0.0 to work. +# so if nevergrad is installed, bayes_opt will not work and vice-versa. +IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2 = ( + _is_installed("bayes_opt") + and importlib.metadata.version("bayesian_optimization") > "2.0.0" +) IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED = _is_installed("gradient_free_optimizers") IS_PYGAD_INSTALLED = _is_installed("pygad") diff --git a/src/optimagic/optimizers/bayesian_optimizer.py b/src/optimagic/optimizers/bayesian_optimizer.py index f80c5fc01..0915c52b5 100644 --- a/src/optimagic/optimizers/bayesian_optimizer.py +++ b/src/optimagic/optimizers/bayesian_optimizer.py @@ -10,7 +10,7 @@ from scipy.optimize import NonlinearConstraint from optimagic import mark -from optimagic.config import IS_BAYESOPT_INSTALLED +from optimagic.config import IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2 from optimagic.exceptions import NotInstalledError from optimagic.optimization.algo_options import N_RESTARTS from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult @@ -35,7 +35,7 @@ @mark.minimizer( name="bayes_opt", solver_type=AggregationLevel.SCALAR, - is_available=IS_BAYESOPT_INSTALLED, + is_available=IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, is_global=True, needs_jac=False, needs_hess=False, @@ -205,7 +205,7 @@ class BayesOpt(Algorithm): def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] ) -> InternalOptimizeResult: - if not IS_BAYESOPT_INSTALLED: + if not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2: raise NotInstalledError( "To use the 'bayes_opt' optimizer you need to install bayes_opt. " "Use 'pip install bayesian-optimization'. " diff --git a/src/optimagic/optimizers/nevergrad_optimizers.py b/src/optimagic/optimizers/nevergrad_optimizers.py index 16166b0a9..22abcfc56 100644 --- a/src/optimagic/optimizers/nevergrad_optimizers.py +++ b/src/optimagic/optimizers/nevergrad_optimizers.py @@ -1,14 +1,17 @@ """Implement optimizers from the nevergrad package.""" +from __future__ import annotations + import math from dataclasses import dataclass +from enum import Enum from typing import TYPE_CHECKING, Any, Literal import numpy as np from numpy.typing import NDArray from optimagic import mark -from optimagic.config import IS_NEVERGRAD_INSTALLED +from optimagic.config import IS_BAYESOPTIM_INSTALLED, IS_NEVERGRAD_INSTALLED from optimagic.exceptions import NotInstalledError from optimagic.optimization.algo_options import ( CONVERGENCE_FTOL_ABS, @@ -30,7 +33,7 @@ ) if TYPE_CHECKING: - import nevergrad as ng + from nevergrad.optimization.base import ConfiguredOptimizer NEVERGRAD_NOT_INSTALLED_ERROR = ( @@ -58,18 +61,84 @@ ) @dataclass(frozen=True) class NevergradPSO(Algorithm): + """Minimize a scalar function using the Particle Swarm Optimization algorithm. + + The Particle Swarm Optimization algorithm was originally proposed by + :cite:`Kennedy1995`.The implementation in Nevergrad is based on + :cite:`Zambrano2013`. + + PSO solves an optimization problem by evolving a swarm of particles + (candidate solutions) across the search space. Each particle adjusts its position + based on its own experience (cognitive component) and the experiences + of its neighbors or the swarm (social component), using velocity updates. The + algorithm iteratively guides the swarm toward promising regions of the search + space. + + """ + transform: Literal["arctan", "gaussian", "identity"] = "arctan" + """The transform used to map from PSO optimization space to real space.""" + population_size: int | None = None + """The number of particles in the swarm.""" + n_cores: int = 1 + """The number of CPU cores to use for parallel computation.""" + seed: int | None = None + """Random seed for reproducibility.""" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations.""" + inertia: float = 0.5 / math.log(2.0) + r"""Inertia weight ω. + + Controls the influence of a particle's previous velocity. Must be less than 1 to + avoid divergence. + + """ + cognitive: float = 0.5 + math.log(2.0) + r"""Cognitive coefficient :math:`\phi_p`. + + Controls the influence of a particle's own best known position. Typical values: 1.0 + to 3.0. + + """ + social: float = 0.5 + math.log(2.0) + r"""Social coefficient. + + Denoted by :math:`\phi_g`. Controls the influence of the swarm's best known + position. Typical values: 1.0 to 3.0. + + """ + quasi_opp_init: bool = False + """Whether to use quasi-opposition initialization. + + Default is False. + + """ + speed_quasi_opp_init: bool = False + """Whether to apply quasi-opposition initialization to speed. + + Default is False. + + """ + special_speed_quasi_opp_init: bool = False + """Whether to use special quasi-opposition initialization for speed. + + Default is False. + + """ + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²) in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -121,40 +190,154 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradCMAES(Algorithm): + """Minimize a scalar function using the Covariance Matrix Adaptation Evolution + Strategy (CMA-ES) algorithm. + + The CMA-ES is a state-of-the-art evolutionary algorithm for difficult non-linear, + non-convex, black-box optimization problems in continuous domains. It is typically + applied to unconstrained or bounded problems with dimensionality between 3 and 100. + CMA-ES adapts a multivariate normal distribution to approximate the objective + function's shape by estimating a positive-definite covariance matrix, akin to the + inverse Hessian in convex-quadratic problems, but without requiring derivatives. + + This implementation is a python wrapper over the original code. + + Original paper can be accessed at `cma-es + `_. + + """ + scale: NonNegativeFloat = 1.0 + """Scale of the search.""" + elitist: bool = False + """Whether to switch to elitist mode (also known as (μ,λ)-CMA-ES). + + In elitist mode, the best point in the population is always retained. + + """ + population_size: int | None = None + """Population size.""" + diagonal: bool = False + """Use the diagonal version of CMA, which is more efficient for high-dimensional + problems.""" + high_speed: bool = False + """Use a metamodel for recommendation to speed up optimization.""" + fast_cmaes: bool = False + """Use the fast CMA-ES implementation. + + Cannot be used with diagonal=True. Produces equivalent results and is preferable for + high dimensions or when objective function evaluations are fast. + + """ + random_init: bool = False + """If True, initialize the optimizer with random parameters.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + step_size_adaptive: bool | str = True + """Whether to adapt the step size. + + Can be a boolean or a string specifying the adaptation strategy. + + """ + CSA_dampfac: PositiveFloat = 1.0 + """Damping factor for step size adaptation.""" + CMA_dampsvec_fade: PositiveFloat = 0.1 + """Damping rate for step size adaptation.""" + CSA_squared: bool = False + """Whether to use squared step sizes in updates.""" + CMA_on: float = 1.0 + """Learning rate for the covariance matrix update.""" + CMA_rankone: float = 1.0 + """Multiplier for the rank-one update learning rate of the covariance matrix.""" + CMA_rankmu: float = 1.0 + """Multiplier for the rank-mu update learning rate of the covariance matrix.""" + CMA_cmean: float = 1.0 + """Learning rate for the mean update.""" + CMA_diagonal_decoding: float = 0.0 + """Learning rate for the diagonal update.""" + num_parents: int | None = None + """Number of parents (μ) for recombination.""" + CMA_active: bool = True + """Whether to use negative updates for the covariance matrix.""" + CMA_mirrormethod: Literal[0, 1, 2] = 2 + """Strategy for mirror sampling. + + 0: Unconditional, 1: Selective, 2: Selective + with delay. + + """ + CMA_const_trace: bool | Literal["arithm", "geom", "aeig", "geig"] = False + """How to normalize the trace of the covariance matrix. + + False: No normalization, + True: Normalize to 1. Other options: 'arithm', 'geom', 'aeig', 'geig'. + + """ + CMA_diagonal: int | bool = False + """Number of iterations to use diagonal covariance matrix before switching to full + matrix. + + If False, always use full matrix. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + stopping_maxiter: PositiveInt = STOPPING_MAXITER + """Maximum number of iterations before termination.""" + stopping_maxtime: PositiveFloat = float("inf") + """Maximum time in seconds before termination.""" + stopping_cov_mat_cond: NonNegativeFloat = 1e14 + """Maximum condition number of the covariance matrix before termination.""" + convergence_ftol_abs: NonNegativeFloat = CONVERGENCE_FTOL_ABS + """Absolute tolerance on function value changes for convergence.""" + convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL + """Relative tolerance on function value changes for convergence.""" + convergence_xtol_abs: NonNegativeFloat = CONVERGENCE_XTOL_ABS + """Absolute tolerance on parameter changes for convergence.""" + convergence_iter_noimprove: PositiveInt | None = None + """Number of iterations without improvement before termination.""" + invariant_path: bool = False + """Whether evolution path (pc) should be invariant to transformations.""" + eval_final_mean: bool = True + """Whether to evaluate the final mean solution.""" + seed: int | None = None + """Seed used by the internal random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -231,11 +414,34 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradOnePlusOne(Algorithm): + """Minimize a scalar function using the One-Plus-One Evolutionary algorithm. + + The One-Plus-One evolutionary algorithm iterates to find a set of parameters + that minimizes the loss function. It does this by perturbing, or mutating, + the parameters from the last iteration (the parent). If the new (child) + parameters yield a better result, the child becomes the new parent whose + parameters are perturbed, perhaps more aggressively. If the parent yields a + better result, it remains the parent and the next perturbation is less + aggressive. + + Originally proposed by :cite:`Rechenberg1973`. The implementation in + Nevergrad is based on the one-fifth adaptation rule from :cite:`Schumer1968`. + + """ + noise_handling: ( Literal["random", "optimistic"] | tuple[Literal["random", "optimistic"], float] | None ) = None + """Method for handling noise. + + 'random' reevaluates a random point, while 'optimistic' reevaluates the best + optimistic point. A float coefficient can be provided to tune the regularity of + these reevaluations. + + """ + mutation: Literal[ "gaussian", "cauchy", @@ -261,27 +467,75 @@ class NevergradOnePlusOne(Algorithm): "biglognormal", "hugelognormal", ] = "gaussian" + """Type of mutation to apply. + + 'gaussian' is the default. Other options include 'cauchy', 'discrete', 'fastga', + 'rls', and 'portfolio'. + + """ + annealing: ( Literal[ "none", "Exp0.9", "Exp0.99", "Exp0.9Auto", "Lin100.0", "Lin1.0", "LinAuto" ] | None ) = None + """Annealing schedule for mutation amplitude. + + Can be 'none', exponential (e.g., 'Exp0.9'), or linear (e.g., 'Lin100.0'). + + """ + sparse: bool = False + """Whether to apply random mutations that set variables to zero.""" + super_radii: bool = False + """Whether to apply extended radii beyond standard bounds for candidate generation, + enabling broader exploration.""" + smoother: bool = False + """Whether to suggest smooth mutations.""" + roulette_size: PositiveInt = 64 + """Size of the roulette wheel used for selection, affecting sampling diversity from + past candidates.""" + antismooth: NonNegativeInt = 4 + """Degree of anti-smoothing to prevent premature convergence by penalizing overly + smooth improvements.""" + crossover: bool = False + """Whether to include a genetic crossover step every other iteration.""" + crossover_type: ( Literal["none", "rand", "max", "min", "onepoint", "twopoint"] | None ) = None + """Method for genetic crossover. + + Options include 'rand', 'onepoint', and 'twopoint'. + + """ + tabu_length: NonNegativeInt = 1000 + """Length of the tabu list to prevent revisiting recent candidates and help escape + local minima.""" + rotation: bool = False + """Whether to apply rotational transformations to the search space to enhance search + performance.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel computation.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)if bounds are not + provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -336,13 +590,32 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradDifferentialEvolution(Algorithm): + """Minimize a scalar function using the Differential Evolution optimizer. + + Differential Evolution is typically used for continuous optimization. It uses + differences between points in the population for performing mutations in fruitful + directions. It is a kind of covariance adaptation without any explicit covariance, + making it very fast in high dimensions. + + """ + initialization: Literal["parametrization", "LHS", "QR", "QO", "SO"] = ( "parametrization" ) + """Algorithm for initialization. + + 'LHS' is Latin Hypercube Sampling, 'QR' is Quasi-Random. + + """ + scale: float | str = 1.0 + """Scale of random component of updates.""" + recommendation: Literal["pessimistic", "optimistic", "mean", "noisy"] = ( "pessimistic" ) + """Criterion for selecting the best point to recommend.""" + crossover: ( float | Literal[ @@ -354,14 +627,41 @@ class NevergradDifferentialEvolution(Algorithm): "parametrization", ] ) = 0.5 + """Crossover rate or strategy. + + Can be a float, 'dimension' (1/dim), 'random', 'onepoint', or 'twopoints'. + + """ + F1: PositiveFloat = 0.8 + """Differential weight #1 (scaling factor).""" + F2: PositiveFloat = 0.8 + """Differential weight #2 (scaling factor).""" + population_size: int | Literal["standard", "dimension", "large"] = "standard" + """Population size. + + Can be an integer or a string like 'standard', 'dimension', or 'large' to set it + automatically. + + """ + high_speed: bool = False + """If True, uses a metamodel for recommendations to speed up optimization.""" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)if bounds are not + provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -372,6 +672,7 @@ def _solve_internal_problem( import nevergrad as ng configured_optimizer = ng.optimizers.DifferentialEvolution( + initialization=self.initialization, scale=self.scale, recommendation=self.recommendation, crossover=self.crossover, @@ -397,7 +698,7 @@ def _solve_internal_problem( @mark.minimizer( name="nevergrad_bo", solver_type=AggregationLevel.SCALAR, - is_available=IS_NEVERGRAD_INSTALLED, + is_available=IS_NEVERGRAD_INSTALLED and IS_BAYESOPTIM_INSTALLED, is_global=True, needs_jac=False, needs_hess=False, @@ -411,14 +712,43 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradBayesOptim(Algorithm): + """Minimize a scalar function using the Bayesian Optimization (BO) algorithm. + + This wrapper uses the BO and PCA-BO algorithms from the `bayes_optim` package + :cite:`bayesoptimimpl`. PCA-BO (Principal Component Analysis for Bayesian + Optimization) is a dimensionality reduction technique for black-box + optimization. It applies PCA to the input space before performing Bayesian + optimization, improving efficiency in high dimensions by focusing on + directions of greatest variance. + + """ + init_budget: int | None = None + """Number of initialization algorithm steps.""" + pca: bool = False + """Whether to use the PCA transformation, defining PCA-BO rather than standard + BO.""" + n_components: NonNegativeFloat = 0.95 + """Number of principal axes, representing the percentage of explained variance + (e.g., 0.95 means 95% variance retained).""" + prop_doe_factor: NonNegativeFloat | None = 1 + """Percentage of the initial budget used for Design of Experiments (DoE).""" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: int | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -465,14 +795,54 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradEMNA(Algorithm): + """Minimize a scalar function using the Estimation of Multivariate Normal Algorithm. + + EMNA is a distribution-based evolutionary algorithm that models the search + space using a multivariate Gaussian. It learns the full covariance matrix, + resulting in a cubic time complexity with respect to each sampling. It is + efficient in parallel settings but other methods should be considered first. + See :cite:`emnaimpl`. + + """ + isotropic: bool = True + """If True, uses an isotropic (identity covariance) Gaussian. + + If False, uses a separable (diagonal covariance) Gaussian. + + """ + noise_handling: bool = True + """If True, returns the best individual found. + + If False (recommended for noisy problems), returns the average of the final + population. + + """ + population_size_adaptation: bool = False + """If True, the population size is adjusted automatically based on the optimization + landscape and noise level.""" + initial_popsize: int | None = None + """Initial population size. + + Defaults to 4 times the problem dimension. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -519,10 +889,27 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradCGA(Algorithm): + """Minimize a scalar function using the Compact Genetic Algorithm. + + The Compact Genetic Algorithm (cGA) is a memory-efficient genetic algorithm + that represents the population as a probability vector over gene values. It + simulates the behavior of a simple GA with uniform crossover by updating + probabilities instead of maintaining an explicit population. See :cite:`cgaimpl`. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -564,10 +951,28 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradEDA(Algorithm): + """Minimize a scalar function using the Estimation of Distribution Algorithm. + + Estimation of Distribution Algorithms (EDAs) optimize by building and sampling + a probabilistic model of promising solutions. Instead of using traditional + variation operators like crossover or mutation, EDAs update a distribution + based on selected individuals and sample new candidates from it. + Refer to :cite:`edaimpl`. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -609,12 +1014,43 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradTBPSA(Algorithm): + """Minimize a scalar function using the Test-based Population Size Adaptation + algorithm. + + TBPSA adapts population size based on fitness trend detection using linear + regression. If no significant improvement is found (via hypothesis testing), + the population size is increased to improve robustness, making it effective + for noisy optimization problems. For more details, refer to :cite:`tbpsaimpl`. + + """ + noise_handling: bool = True + """If True, returns the best individual. + + If False (recommended for noisy problems), returns the average of the final + population to reduce noise. + + """ + initial_popsize: int | None = None + """Initial population size. + + If not specified, defaults to 4 times the problem dimension. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -659,16 +1095,52 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradRandomSearch(Algorithm): + """Minimize a scalar function using the Random Search algorithm. + + This is a one-shot optimization method that provides random suggestions and serves + as a simple baseline for other optimizers. + + """ + middle_point: bool = False + """Enforces that the first suggested point is the zero vector.""" + opposition_mode: Literal["opposite", "quasi"] | None = None + """Symmetrizes exploration with respect to the center. + + 'opposite' enables full symmetry, while 'quasi' applies randomized symmetry. + + """ + sampler: Literal["parametrization", "gaussian", "cauchy"] = "parametrization" + """The probability distribution for sampling points. + + 'gaussian' and 'cauchy' are available alternatives. + + """ + scale: PositiveFloat | Literal["random", "auto", "autotune"] = "auto" + """Scalar used to multiply suggested point values. + + Can be a float or a string for auto-scaling ('random', 'auto', 'autotune'). + + """ + recommendation_rule: Literal[ "average_of_best", "pessimistic", "average_of_exp_best" ] = "pessimistic" + """Specifies how the final recommendation is chosen, e.g., 'pessimistic' (default) + or 'average_of_best'.""" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -717,17 +1189,60 @@ def _solve_internal_problem( ) @dataclass(frozen=True) class NevergradSamplingSearch(Algorithm): + """Minimize a scalar function using SamplingSearch. + + This is a one-shot optimization method that is better than random search because it + uses low-discrepancy sequences to ensure more uniform coverage of the search space. + It is recommended to use "Hammersley" as the sampler if the budget is known, and to + set `scrambled=True` in high dimensions. + + """ + sampler: Literal["Halton", "LHS", "Hammersley"] = "Halton" + """Choice of the low-discrepancy sampler used for generating points. + + 'LHS' is Latin Hypercube Sampling. + + """ + scrambled: bool = False + """If True, adds scrambling to the search sequence, which is highly recommended for + high-dimensional problems.""" + middle_point: bool = False + """If True, the first suggested point is the zero vector, useful for initializing at + the center of the search space.""" + cauchy: bool = False + """If True, uses the inverse Cauchy distribution instead of Gaussian when projecting + samples to a real-valued space.""" + scale: bool | NonNegativeFloat = 1.0 + """A float multiplier to scale all generated points.""" + rescaled: bool = False + """If True, rescales the sampling pattern to ensure better coverage of the + boundaries.""" + recommendation_rule: Literal["average_of_best", "pessimistic"] = "pessimistic" + """How the final recommendation is chosen. + + 'pessimistic' is the default. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -753,15 +1268,79 @@ def _solve_internal_problem( configured_optimizer=configured_optimizer, stopping_maxfun=self.stopping_maxfun, n_cores=self.n_cores, - seed=None, + seed=self.seed, sigma=self.sigma, nonlinear_constraints=problem.nonlinear_constraints, ) return res +# TODO https://facebookresearch.github.io/nevergrad/optimizers_ref.html#nevergrad.families.EvolutionStrategy + + +class Wizard(str, Enum): + """Available portfolio optimizers from Nevergrad.""" + + # REF https://openreview.net/pdf/bcf18ffaccd27991ddf707a37b164dbab4ec4771.pdf + NGOpt = "NGOpt" + NGOpt4 = "NGOpt4" + NGOpt8 = "NGOpt8" + NGOpt10 = "NGOpt10" + NGOpt12 = "NGOpt12" + NGOpt13 = "NGOpt13" + NGOpt14 = "NGOpt14" + NGOpt15 = "NGOpt15" + NGOpt16 = "NGOpt16" + NGOpt21 = "NGOpt21" + NGOpt36 = "NGOpt36" + NGOpt38 = "NGOpt38" + NGOpt39 = "NGOpt39" + NGOptRW = "NGOptRW" + NGOptF = "NGOptF" + NGOptF2 = "NGOptF2" + NGOptF3 = "NGOptF3" + NGOptF5 = "NGOptF5" + + NgIoh2 = "NgIoh2" + NgIoh3 = "NgIoh3" + NgIoh4 = "NgIoh4" + NgIoh5 = "NgIoh5" + NgIoh6 = "NgIoh6" + NgIoh7 = "NgIoh7" + NgIoh11 = "NgIoh11" + NgIoh14 = "NgIoh14" + NgIoh13 = "NgIoh13" + NgIoh15 = "NgIoh15" + NgIoh12 = "NgIoh12" + NgIoh16 = "NgIoh16" + NgIoh17 = "NgIoh17" + NgIoh21 = "NgIoh21" + NgIoh20 = "NgIoh20" + NgIoh19 = "NgIoh19" + NgIoh18 = "NgIoh18" + NgIoh10 = "NgIoh10" + NgIoh9 = "NgIoh9" + NgIoh8 = "NgIoh8" + NgIoh12b = "NgIoh12b" + NgIoh13b = "NgIoh13b" + NgIoh14b = "NgIoh14b" + NgIoh15b = "NgIoh15b" + + NgDS = "NgDS" + NgDS2 = "NgDS2" + NGDSRW = "NGDSRW" + NGO = "NGO" + NgIohRW2 = "NgIohRW2" + NgIohTuned = "NgIohTuned" + + CSEC = "CSEC" + CSEC10 = "CSEC10" + CSEC11 = "CSEC11" + Wiz = "Wiz" + + @mark.minimizer( - name="nevergrad_NGOpt", + name="nevergrad_wizard", solver_type=AggregationLevel.SCALAR, is_available=IS_NEVERGRAD_INSTALLED, is_global=True, @@ -776,65 +1355,34 @@ def _solve_internal_problem( disable_history=False, ) @dataclass(frozen=True) -class NevergradNGOpt(Algorithm): - optimizer: Literal[ - "NGOpt", - "NGOpt4", - "NGOpt8", - "NGOpt10", - "NGOpt12", - "NGOpt13", - "NGOpt14", - "NGOpt15", - "NGOpt16", - "NGOpt21", - "NGOpt36", - "NGOpt38", - "NGOpt39", - "NGOptRW", - "NGOptF", - "NGOptF2", - "NGOptF3", - "NGOptF5", - "NgIoh2", - "NgIoh3", - "NgIoh4", - "NgIoh5", - "NgIoh6", - "NgIoh7", - "NgIoh11", - "NgIoh14", - "NgIoh13", - "NgIoh15", - "NgIoh12", - "NgIoh16", - "NgIoh17", - "NgIoh21", - "NgIoh20", - "NgIoh19", - "NgIoh18", - "NgIoh10", - "NgIoh9", - "NgIoh8", - "NgIoh12b", - "NgIoh13b", - "NgIoh14b", - "NgIoh15b", - "NgDS", - "NgDS2", - "NGDSRW", - "NGO", - "NgIohRW2", - "NgIohTuned", - "CSEC", - "CSEC10", - "CSEC11", - "Wiz", - ] = "NGOpt" +class NevergradWizard(Algorithm): + """Minimize a scalar function using a Meta Optimizer from Nevergrad. + + These are meta-optimizers that intelligently combine multiple different + optimization algorithms to solve a problem. The specific portfolio of + optimizers can be selected via the `optimizer` parameter. + + """ + + optimizer: Wizard = Wizard.NgIoh10 # rename algorithm_selection maybe + """The specific Nevergrad meta-optimizer to use. + + Each option is a portfolio of different algorithms. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²)in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -860,8 +1408,79 @@ def _solve_internal_problem( return res +class Portfolio(str, Enum): + """Available portfolio optimizers in Nevergrad.""" + + Carola1 = "Carola1" + """ + CAROLA1 - Cost-effective Asymptotic Randomized Optimization with Limited Access. + + Method: + + 1. COBYLA (budget b/2). + 2. CMA with Meta Model (budget b/2), starting from COBYLA’s best solution, + """ + Carola2 = "Carola2" + """ + CAROLA2 - see Carola1 + + Method + + 1. COBYLA (budget b/3) for fast approximation. + 2. CMA with meta-model (budget b/3), starting from COBYLA’s best solution, + for robust local search. + 3. SQP (budget b/3), starting from the best solution so far, + for fast refinement. + """ + Carola3 = "Carola3" + """ + CAROLA3 - CAROLA2 for the parallel case. see Carola2, + + Method + 1. Apply w copies of Carola2 in parallel, with budget b/w.""" + + MultiBFGSPlus = "MultiBFGSPlus" + LogMultiBFGSPlus = "LogMultiBFGSPlus" + SqrtMultiBFGSPlus = "SqrtMultiBFGSPlus" + MultiCobylaPlus = "MultiCobylaPlus" + MultiSQPPlus = "MultiSQPPlus" + BFGSCMAPlus = "BFGSCMAPlus" + LogBFGSCMAPlus = "LogBFGSCMAPlus" + SqrtBFGSCMAPlus = "SqrtBFGSCMAPlus" + SQPCMAPlus = "SQPCMAPlus" + LogSQPCMAPlus = "LogSQPCMAPlus" + SqrtSQPCMAPlus = "SqrtSQPCMAPlus" + + MultiBFGS = "MultiBFGS" + LogMultiBFGS = "LogMultiBFGS" + SqrtMultiBFGS = "SqrtMultiBFGS" + MultiCobyla = "MultiCobyla" + ForceMultiCobyla = "ForceMultiCobyla" + MultiSQP = "MultiSQP" + BFGSCMA = "BFGSCMA" + LogBFGSCMA = "LogBFGSCMA" + SqrtBFGSCMA = "SqrtBFGSCMA" + SQPCMA = "SQPCMA" + LogSQPCMA = "LogSQPCMA" + SqrtSQPCMA = "SqrtSQPCMA" + FSQPCMA = "FSQPCMA" + F2SQPCMA = "F2SQPCMA" + F3SQPCMA = "F3SQPCMA" + + MultiDiscrete = "MultiDiscrete" + CMandAS2 = "CMandAS2" + CMandAS3 = "CMandAS3" + MetaCMA = "MetaCMA" + CMA = "CMA" + PCEDA = "PCEDA" + MPCEDA = "MPCEDA" + MEDA = "MEDA" + NoisyBandit = "NoisyBandit" + Shiwa = "Shiwa" + + @mark.minimizer( - name="nevergrad_meta", + name="nevergrad_portfolio", solver_type=AggregationLevel.SCALAR, is_available=IS_NEVERGRAD_INSTALLED, is_global=True, @@ -876,50 +1495,34 @@ def _solve_internal_problem( disable_history=False, ) @dataclass(frozen=True) -class NevergradMeta(Algorithm): - optimizer: Literal[ - "MultiBFGSPlus", - "LogMultiBFGSPlus", - "SqrtMultiBFGSPlus", - "MultiCobylaPlus", - "MultiSQPPlus", - "BFGSCMAPlus", - "LogBFGSCMAPlus", - "SqrtBFGSCMAPlus", - "SQPCMAPlus", - "LogSQPCMAPlus", - "SqrtSQPCMAPlus", - "MultiBFGS", - "LogMultiBFGS", - "SqrtMultiBFGS", - "MultiCobyla", - "ForceMultiCobyla", - "MultiSQP", - "BFGSCMA", - "LogBFGSCMA", - "SqrtBFGSCMA", - "SQPCMA", - "LogSQPCMA", - "SqrtSQPCMA", - "FSQPCMA", - "F2SQPCMA", - "F3SQPCMA", - "MultiDiscrete", - "CMandAS2", - "CMandAS3", - "MetaCMA", - "CMA", - "PCEDA", - "MPCEDA", - "MEDA", - "NoisyBandit", - "Shiwa", - "Carola3", - ] = "Shiwa" +class NevergradPortfolio(Algorithm): + """Minimize a scalar function using a Meta Optimizer from Nevergrad. + + This algorithm utilizes a combination of local and global optimizers to find + the best solution. The specific portfolio of optimizers can be selected via + the `optimizer` parameter. + + """ + + optimizer: Portfolio = Portfolio.BFGSCMA + """The specific Nevergrad meta-optimizer to use. + + Each option is a portfolio of different local and global algorithms. + + """ + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of function evaluations before termination.""" + n_cores: PositiveInt = 1 + """Number of cores to use for parallel function evaluation.""" + seed: int | None = None + """Seed for the random number generator for reproducibility.""" + sigma: float | None = None + """Standard deviation for sampling initial population from N(0, σ²) in case bounds + are not provided.""" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -949,7 +1552,7 @@ def _nevergrad_internal( problem: InternalOptimizationProblem, x0: NDArray[np.float64], n_cores: int, - configured_optimizer: "ng.optimization.base.ConfiguredOptimizer", + configured_optimizer: ConfiguredOptimizer, stopping_maxfun: int, seed: int | None, sigma: float | None, @@ -978,20 +1581,17 @@ def _nevergrad_internal( param = ng.p.Array( init=x0, - ) - - param.set_bounds( lower=problem.bounds.lower, upper=problem.bounds.upper, ) + instrum = ng.p.Instrumentation(param) + # In case bounds are not provided, the initial population is sampled - # from a gaussian with mean = 0 and sigma = 1, + # from a gaussian with mean = 0 and sd = 1, # which can be set through this method. param.set_mutation(sigma=sigma) - instrum = ng.p.Instrumentation(param) - if seed is not None: instrum.random_state.seed(seed) diff --git a/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py b/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py index b83cb51ae..5e8890e28 100644 --- a/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py +++ b/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py @@ -1,5 +1,6 @@ import numpy as np import pytest +from numpy.testing import assert_array_almost_equal as aaae from optimagic import mark from optimagic.config import IS_NEVERGRAD_INSTALLED @@ -22,6 +23,6 @@ def test_no_bounds_with_nevergrad(): algorithm="nevergrad_cmaes", collect_history=True, skip_checks=True, - algo_options={"seed": 12345}, + algo_options={"seed": 12345, "stopping_maxfun": 10000}, ) - print(res) + aaae(res.x, np.zeros(3), 4) diff --git a/tests/optimagic/optimizers/test_bayesian_optimizer.py b/tests/optimagic/optimizers/test_bayesian_optimizer.py index 0e6f739d9..39bab2df5 100644 --- a/tests/optimagic/optimizers/test_bayesian_optimizer.py +++ b/tests/optimagic/optimizers/test_bayesian_optimizer.py @@ -3,10 +3,10 @@ import numpy as np import pytest -from optimagic.config import IS_BAYESOPT_INSTALLED +from optimagic.config import IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2 from optimagic.optimization.internal_optimization_problem import InternalBounds -if IS_BAYESOPT_INSTALLED: +if IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2: from bayes_opt import acquisition from optimagic.optimizers.bayesian_optimizer import ( @@ -49,7 +49,10 @@ def test_process_bounds_infinite(): _process_bounds(bounds) -@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.skipif( + not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, + reason="bayes_opt is not installed in a recent enough version >= 2.0.0.", +) def test_process_acquisition_function_none(): """Test processing None acquisition function.""" result = _process_acquisition_function( @@ -63,7 +66,9 @@ def test_process_acquisition_function_none(): assert result is None -@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.skipif( + not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, reason="bayes_opt not installed" +) @pytest.mark.parametrize( "acq_name, expected_class", [ @@ -88,7 +93,9 @@ def test_process_acquisition_function_string(acq_name, expected_class): assert isinstance(result, expected_class) -@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.skipif( + not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, reason="bayes_opt not installed" +) def test_process_acquisition_function_invalid_string(): """Test processing invalid string acquisition function.""" with pytest.raises(ValueError, match="Invalid acquisition_function string"): @@ -102,7 +109,9 @@ def test_process_acquisition_function_invalid_string(): ) -@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.skipif( + not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, reason="bayes_opt not installed" +) def test_process_acquisition_function_instance(): """Test processing acquisition function instance.""" acq_instance = acquisition.UpperConfidenceBound() @@ -117,7 +126,9 @@ def test_process_acquisition_function_instance(): assert result is acq_instance -@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.skipif( + not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, reason="bayes_opt not installed" +) def test_process_acquisition_function_class(): """Test processing acquisition function class.""" result = _process_acquisition_function( @@ -131,7 +142,9 @@ def test_process_acquisition_function_class(): assert isinstance(result, acquisition.UpperConfidenceBound) -@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.skipif( + not IS_BAYESOPT_INSTALLED_AND_VERSION_NEWER_THAN_2, reason="bayes_opt not installed" +) def test_process_acquisition_function_invalid_type(): """Test processing invalid acquisition function type.""" with pytest.raises(TypeError, match="acquisition_function must be None, a string"): diff --git a/tests/optimagic/optimizers/test_nevergrad.py b/tests/optimagic/optimizers/test_nevergrad.py index af351c005..bd4043284 100644 --- a/tests/optimagic/optimizers/test_nevergrad.py +++ b/tests/optimagic/optimizers/test_nevergrad.py @@ -1,18 +1,15 @@ """Test helper functions for nevergrad optimizers.""" -from typing import get_args +import warnings -import numpy as np -import pytest -from numpy.testing import assert_array_almost_equal as aaae - -from optimagic import algorithms, mark +from optimagic import mark from optimagic.config import IS_NEVERGRAD_INSTALLED -from optimagic.optimization.optimize import minimize -from optimagic.parameters.bounds import Bounds if IS_NEVERGRAD_INSTALLED: - import nevergrad as ng + import cma + + ## Skip warnings during tests + warnings.simplefilter("ignore", cma.evolution_strategy.InjectionWarning) @mark.least_squares @@ -69,67 +66,67 @@ def sos(x): # expected = [[np.array([-2.0]), np.array([-1.0])]] * 2 # assert got == expected ### - - -# test if all optimizers listed in Literal type hint are valid attributes -@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") -def test_meta_optimizers_are_valid(): - opt = algorithms.NevergradMeta - optimizers = get_args(opt.__annotations__["optimizer"]) - for optimizer in optimizers: - try: - getattr(ng.optimizers, optimizer) - except AttributeError: - pytest.fail(f"Optimizer '{optimizer}' not found in Nevergrad") - - -@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") -def test_ngopt_optimizers_are_valid(): - opt = algorithms.NevergradNGOpt - optimizers = get_args(opt.__annotations__["optimizer"]) - for optimizer in optimizers: - try: - getattr(ng.optimizers, optimizer) - except AttributeError: - pytest.fail(f"Optimizer '{optimizer}' not found in Nevergrad") - - -# list of available optimizers in nevergrad_meta -NEVERGRAD_META = get_args(algorithms.NevergradMeta.__annotations__["optimizer"]) -# list of available optimizers in nevergrad_ngopt -NEVERGRAD_NGOPT = get_args(algorithms.NevergradNGOpt.__annotations__["optimizer"]) - - -# test stochastic_global_algorithm_on_sum_of_squares -@pytest.mark.slow -@pytest.mark.parametrize("algorithm", NEVERGRAD_META) -@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") -def test_meta_optimizers_with_stochastic_global_algorithm_on_sum_of_squares(algorithm): - res = minimize( - fun=sos, - params=np.array([0.35, 0.35]), - bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), - algorithm=algorithms.NevergradMeta(algorithm), - collect_history=False, - skip_checks=True, - algo_options={"seed": 12345}, - ) - assert res.success in [True, None] - aaae(res.params, np.array([0.2, 0]), decimal=1) - - -@pytest.mark.slow -@pytest.mark.parametrize("algorithm", NEVERGRAD_NGOPT) -@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") -def test_ngopt_optimizers_with_stochastic_global_algorithm_on_sum_of_squares(algorithm): - res = minimize( - fun=sos, - params=np.array([0.35, 0.35]), - bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), - algorithm=algorithms.NevergradNGOpt(algorithm), - collect_history=False, - skip_checks=True, - algo_options={"seed": 12345}, - ) - assert res.success in [True, None] - aaae(res.params, np.array([0.2, 0]), decimal=1) +################################################################################### + +# # test if all optimizers listed in Literal type hint are valid attributes +# @pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +# def test_meta_optimizers_are_valid(): +# opt = algorithms.NevergradMeta +# optimizers = get_args(opt.__annotations__["optimizer"]) +# for optimizer in optimizers: +# try: +# getattr(ng.optimizers, optimizer) +# except AttributeError: +# pytest.fail(f"Optimizer '{optimizer}' not found in Nevergrad") + + +# @pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +# def test_ngopt_optimizers_are_valid(): +# opt = algorithms.NevergradNGOpt +# optimizers = get_args(opt.__annotations__["optimizer"]) +# for optimizer in optimizers: +# try: +# getattr(ng.optimizers, optimizer) +# except AttributeError: +# pytest.fail(f"Optimizer '{optimizer}' not found in Nevergrad") + + +# # list of available optimizers in nevergrad_meta +# NEVERGRAD_META = get_args(algorithms.NevergradMeta.__annotations__["optimizer"]) +# # list of available optimizers in nevergrad_ngopt +# NEVERGRAD_NGOPT = get_args(algorithms.NevergradNGOpt.__annotations__["optimizer"]) + + +# # test stochastic_global_algorithm_on_sum_of_squares +# @pytest.mark.slow +# @pytest.mark.parametrize("algorithm", NEVERGRAD_META) +# @pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +# def test_meta_optimizers_with_stochastic_global_algorithm_on_sos(algorithm): +# res = minimize( +# fun=sos, +# params=np.array([0.35, 0.35]), +# bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), +# algorithm=algorithms.NevergradMeta(algorithm), +# collect_history=False, +# skip_checks=True, +# algo_options={"seed": 12345}, +# ) +# assert res.success in [True, None] +# aaae(res.params, np.array([0.2, 0]), decimal=1) + + +# @pytest.mark.slow +# @pytest.mark.parametrize("algorithm", NEVERGRAD_NGOPT) +# @pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +# def test_ngopt_optimizers_with_stochastic_global_algorithm_on_sos(algorithm): +# res = minimize( +# fun=sos, +# params=np.array([0.35, 0.35]), +# bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), +# algorithm=algorithms.NevergradNGOpt(algorithm), +# collect_history=False, +# skip_checks=True, +# algo_options={"seed": 12345}, +# ) +# assert res.success in [True, None] +# aaae(res.params, np.array([0.2, 0]), decimal=1)