Coverage for mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Test multi-target optimization.""" 

6 

7import logging 

8from typing import List, Optional, Type 

9 

10import ConfigSpace as CS 

11import pandas as pd 

12import pytest 

13 

14from mlos_core.data_classes import Observations, Suggestion 

15from mlos_core.optimizers import BaseOptimizer, OptimizerType 

16from mlos_core.tests import SEED 

17 

18_LOG = logging.getLogger(__name__) 

19 

20 

21@pytest.mark.parametrize( 

22 ("optimizer_class", "kwargs"), 

23 [ 

24 *[(member.value, {}) for member in OptimizerType], 

25 ], 

26) 

27def test_multi_target_opt_wrong_weights( 

28 optimizer_class: Type[BaseOptimizer], 

29 kwargs: dict, 

30) -> None: 

31 """Make sure that the optimizer raises an error if the number of objective weights 

32 does not match the number of optimization targets. 

33 """ 

34 with pytest.raises(ValueError): 

35 optimizer_class( 

36 parameter_space=CS.ConfigurationSpace(seed=SEED), 

37 optimization_targets=["main_score", "other_score"], 

38 objective_weights=[1], 

39 **kwargs, 

40 ) 

41 

42 

43@pytest.mark.parametrize( 

44 ("objective_weights"), 

45 [ 

46 [2, 1], 

47 [0.5, 0.5], 

48 None, 

49 ], 

50) 

51@pytest.mark.parametrize( 

52 ("optimizer_class", "kwargs"), 

53 [ 

54 *[(member.value, {}) for member in OptimizerType], 

55 ], 

56) 

57def test_multi_target_opt( 

58 objective_weights: Optional[List[float]], 

59 optimizer_class: Type[BaseOptimizer], 

60 kwargs: dict, 

61) -> None: 

62 """Toy multi-target optimization problem to test the optimizers with mixed numeric 

63 types to ensure that original dtypes are retained. 

64 """ 

65 # pylint: disable=too-many-locals 

66 max_iterations = 10 

67 

68 def objective(point: pd.Series) -> pd.Series: 

69 # mix of hyperparameters, optimal is to select the highest possible 

70 ret: pd.Series = pd.Series( 

71 { 

72 "main_score": point.x + point.y, 

73 "other_score": point.x**2 + point.y**2, 

74 } 

75 ) 

76 return ret 

77 

78 input_space = CS.ConfigurationSpace(seed=SEED) 

79 # add a mix of numeric datatypes 

80 input_space.add(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) 

81 input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) 

82 

83 optimizer = optimizer_class( 

84 parameter_space=input_space, 

85 optimization_targets=["main_score", "other_score"], 

86 objective_weights=objective_weights, 

87 **kwargs, 

88 ) 

89 

90 with pytest.raises(ValueError, match="No observations"): 

91 optimizer.get_best_observations() 

92 

93 with pytest.raises(ValueError, match="No observations"): 

94 optimizer.get_observations() 

95 

96 for _ in range(max_iterations): 

97 suggestion = optimizer.suggest() 

98 assert isinstance(suggestion, Suggestion) 

99 assert isinstance(suggestion.config, pd.Series) 

100 assert suggestion.metadata is None or isinstance(suggestion.metadata, pd.Series) 

101 assert set(suggestion.config.index) == {"x", "y"} 

102 # Check suggestion values are the expected dtype 

103 assert isinstance(suggestion.config["x"], int) 

104 assert isinstance(suggestion.config["y"], float) 

105 # Check that suggestion is in the space 

106 config_dict: dict = suggestion.config.to_dict() 

107 test_configuration = CS.Configuration(optimizer.parameter_space, config_dict) 

108 # Raises an error if outside of configuration space 

109 test_configuration.check_valid_configuration() 

110 # Test registering the suggested configuration with a score. 

111 observation = objective(suggestion.config) 

112 assert isinstance(observation, pd.Series) 

113 assert set(observation.index) == {"main_score", "other_score"} 

114 optimizer.register(observations=suggestion.complete(observation)) 

115 

116 best_observations = optimizer.get_best_observations() 

117 assert isinstance(best_observations, Observations) 

118 assert isinstance(best_observations.configs, pd.DataFrame) 

119 assert isinstance(best_observations.scores, pd.DataFrame) 

120 assert best_observations.contexts is None 

121 assert set(best_observations.configs.columns) == {"x", "y"} 

122 assert set(best_observations.scores.columns) == {"main_score", "other_score"} 

123 assert best_observations.configs.shape == (1, 2) 

124 assert best_observations.scores.shape == (1, 2) 

125 

126 all_observations = optimizer.get_observations() 

127 assert isinstance(all_observations, Observations) 

128 assert isinstance(all_observations.configs, pd.DataFrame) 

129 assert isinstance(all_observations.scores, pd.DataFrame) 

130 assert all_observations.contexts is None 

131 assert set(all_observations.configs.columns) == {"x", "y"} 

132 assert set(all_observations.scores.columns) == {"main_score", "other_score"} 

133 assert all_observations.configs.shape == (max_iterations, 2) 

134 assert all_observations.scores.shape == (max_iterations, 2)