From f57a7150cd774b992c9021c9ccef0a05624748c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Sat, 14 Jun 2025 18:23:27 +0200 Subject: [PATCH 1/3] isolate sklearn --- pyproject.toml | 2 +- .../optimizers/exp_opt/ensemble_optimizer.py | 31 ++++++++++--------- .../tree_structured_parzen_estimators.py | 3 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1a21c4c..32d9bdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ dependencies = [ "scipy <2.0.0", "numpy >=1.18.1, <3.0.0", "pandas <3.0.0", - "scikit-learn >=0.21, !=0.23.*", "tqdm >=4.48.0, <5.0.0", ] @@ -53,6 +52,7 @@ test = [ "pytest == 8.3.5", "flake8", "pytest-cov", + "scikit-learn >=0.21, !=0.23.*", "surfaces", ] diff --git a/src/gradient_free_optimizers/optimizers/exp_opt/ensemble_optimizer.py b/src/gradient_free_optimizers/optimizers/exp_opt/ensemble_optimizer.py index 5efc26c..68339c8 100644 --- a/src/gradient_free_optimizers/optimizers/exp_opt/ensemble_optimizer.py +++ b/src/gradient_free_optimizers/optimizers/exp_opt/ensemble_optimizer.py @@ -3,20 +3,12 @@ # License: MIT License import numpy as np -from scipy.stats import norm from ..smb_opt.smbo import SMBO from ..smb_opt.surrogate_models import EnsembleRegressor from ..smb_opt.acquisition_function import ExpectedImprovement -from sklearn.tree import DecisionTreeRegressor -from sklearn.ensemble import GradientBoostingRegressor -from sklearn.svm import SVR -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.neural_network import MLPRegressor - - def normalize(array): num = array - array.min() den = array.max() - array.min() @@ -41,12 +33,7 @@ def __init__( epsilon=0.03, distribution="normal", n_neighbours=3, - estimators=[ - GradientBoostingRegressor(n_estimators=5), - # DecisionTreeRegressor(), - # MLPRegressor(), - GaussianProcessRegressor(), - ], + estimators=None, xi=0.01, warm_start_smbo=None, max_sample_size=10000000, @@ -71,13 +58,27 @@ def __init__( replacement=replacement, ) self.estimators = estimators - self.regr = EnsembleRegressor(estimators) self.xi = xi self.warm_start_smbo = warm_start_smbo self.max_sample_size = max_sample_size self.sampling = sampling self.warnings = warnings + if estimators is None: + from sklearn.ensemble import GradientBoostingRegressor + from sklearn.gaussian_process import GaussianProcessRegressor + + self._estimators = [ + GradientBoostingRegressor(n_estimators=5), + # DecisionTreeRegressor(), + # MLPRegressor(), + GaussianProcessRegressor(), + ] + else: + self._estimators = estimators + + self.regr = EnsembleRegressor(self._estimators) + self.init_warm_start_smbo() def finish_initialization(self): diff --git a/src/gradient_free_optimizers/optimizers/smb_opt/tree_structured_parzen_estimators.py b/src/gradient_free_optimizers/optimizers/smb_opt/tree_structured_parzen_estimators.py index fd67f2e..8c87ff9 100644 --- a/src/gradient_free_optimizers/optimizers/smb_opt/tree_structured_parzen_estimators.py +++ b/src/gradient_free_optimizers/optimizers/smb_opt/tree_structured_parzen_estimators.py @@ -5,7 +5,6 @@ import numpy as np -from sklearn.neighbors import KernelDensity from .smbo import SMBO @@ -46,6 +45,8 @@ def __init__( self.gamma_tpe = gamma_tpe + from sklearn.neighbors import KernelDensity + kde_para = { "kernel": "gaussian", "bandwidth": 1, From 8986b88e233466316f3872377b6d44f745a45c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Sat, 14 Jun 2025 18:27:01 +0200 Subject: [PATCH 2/3] Update bayesian_optimization.py --- .../optimizers/smb_opt/bayesian_optimization.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py b/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py index d88c71c..39e0763 100644 --- a/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py +++ b/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py @@ -3,7 +3,6 @@ # License: MIT License import numpy as np -from scipy.stats import norm from .smbo import SMBO From a842beb659f60aa7356a5b722c30eda3aac91486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Sat, 14 Jun 2025 18:38:13 +0200 Subject: [PATCH 3/3] gpr --- .../optimizer_search/bayesian_optimization.py | 3 +-- .../smb_opt/bayesian_optimization.py | 18 ++++++++++-------- .../optimizers/smb_opt/forest_optimizer.py | 2 +- .../optimizers/smb_opt/smbo.py | 13 +++++++++---- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/gradient_free_optimizers/optimizer_search/bayesian_optimization.py b/src/gradient_free_optimizers/optimizer_search/bayesian_optimization.py index 443bc6a..f1812c0 100644 --- a/src/gradient_free_optimizers/optimizer_search/bayesian_optimization.py +++ b/src/gradient_free_optimizers/optimizer_search/bayesian_optimization.py @@ -6,7 +6,6 @@ from ..search import Search from ..optimizers import BayesianOptimizer as _BayesianOptimizer -from ..optimizers.smb_opt.bayesian_optimization import gaussian_process class BayesianOptimizer(_BayesianOptimizer, Search): @@ -61,7 +60,7 @@ def __init__( max_sample_size: int = 10000000, sampling: Dict[Literal["random"], int] = {"random": 1000000}, replacement: bool = True, - gpr=gaussian_process["gp_nonlinear"], + gpr=None, xi: float = 0.03, ): super().__init__( diff --git a/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py b/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py index 39e0763..72288bf 100644 --- a/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py +++ b/src/gradient_free_optimizers/optimizers/smb_opt/bayesian_optimization.py @@ -6,14 +6,10 @@ from .smbo import SMBO -from .surrogate_models import ( - GPR_linear, - GPR, -) from .acquisition_function import ExpectedImprovement -gaussian_process = {"gp_nonlinear": GPR(), "gp_linear": GPR_linear()} +# gaussian_process = {"gp_nonlinear": GPR(), "gp_linear": GPR_linear()} def normalize(array): @@ -45,9 +41,9 @@ def __init__( nth_process=None, warm_start_smbo=None, max_sample_size=10000000, - sampling={"random": 1000000}, + sampling=None, replacement=True, - gpr=gaussian_process["gp_nonlinear"], + gpr=None, xi=0.03, ): super().__init__( @@ -64,7 +60,13 @@ def __init__( ) self.gpr = gpr - self.regr = gpr + if gpr is None: + from gradient_free_optimizers.optimizers.smb_opt.surrogate_models import GPR + self._gpr = GPR() + else: + self._gpr = gpr + + self.regr = self._gpr self.xi = xi def finish_initialization(self): diff --git a/src/gradient_free_optimizers/optimizers/smb_opt/forest_optimizer.py b/src/gradient_free_optimizers/optimizers/smb_opt/forest_optimizer.py index 02f2b1c..98bc7d9 100644 --- a/src/gradient_free_optimizers/optimizers/smb_opt/forest_optimizer.py +++ b/src/gradient_free_optimizers/optimizers/smb_opt/forest_optimizer.py @@ -53,7 +53,7 @@ def __init__( nth_process=None, warm_start_smbo=None, max_sample_size=10000000, - sampling={"random": 1000000}, + sampling=None, replacement=True, tree_regressor="extra_tree", tree_para={"n_estimators": 100}, diff --git a/src/gradient_free_optimizers/optimizers/smb_opt/smbo.py b/src/gradient_free_optimizers/optimizers/smb_opt/smbo.py index 66d7f24..eea926f 100644 --- a/src/gradient_free_optimizers/optimizers/smb_opt/smbo.py +++ b/src/gradient_free_optimizers/optimizers/smb_opt/smbo.py @@ -23,7 +23,7 @@ def __init__( nth_process=None, warm_start_smbo=None, max_sample_size=10000000, - sampling={"random": 1000000}, + sampling=None, replacement=True, ): super().__init__( @@ -40,6 +40,11 @@ def __init__( self.sampling = sampling self.replacement = replacement + if sampling is None: + self._sampling_dict = {"random": 1000000} + else: + self._sampling_dict = sampling + self.sampler = InitialSampler(self.conv, max_sample_size) self.init_warm_start_smbo(warm_start_smbo) @@ -97,13 +102,13 @@ def wrapper(self, score): return wrapper def _sampling(self, all_pos_comb): - if self.sampling is False: + if self._sampling_dict is False: return all_pos_comb - elif "random" in self.sampling: + elif "random" in self._sampling_dict: return self.random_sampling(all_pos_comb) def random_sampling(self, pos_comb): - n_samples = self.sampling["random"] + n_samples = self._sampling_dict["random"] n_pos_comb = pos_comb.shape[0] if n_pos_comb <= n_samples: