Coverage for mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py: 100%

43 statements  

« prev     ^ index     » next       coverage.py v7.6.9, created at 2024-12-20 00:44 +0000

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for Tunable to ConfigSpace conversion.""" 

6 

7import pytest 

8from ConfigSpace import ( 

9 CategoricalHyperparameter, 

10 ConfigurationSpace, 

11 EqualsCondition, 

12 Integer, 

13 UniformFloatHyperparameter, 

14 UniformIntegerHyperparameter, 

15) 

16 

17from mlos_bench.optimizers.convert_configspace import ( 

18 TunableValueKind, 

19 _tunable_to_configspace, 

20 special_param_names, 

21 tunable_groups_to_configspace, 

22) 

23from mlos_bench.tunables.tunable import Tunable 

24from mlos_bench.tunables.tunable_groups import TunableGroups 

25from mlos_core.spaces.converters.util import ( 

26 QUANTIZATION_BINS_META_KEY, 

27 monkey_patch_cs_quantization, 

28) 

29 

30# pylint: disable=redefined-outer-name 

31 

32 

33@pytest.fixture 

34def configuration_space() -> ConfigurationSpace: 

35 """ 

36 A test fixture that produces a mock ConfigurationSpace object matching the 

37 tunable_groups fixture. 

38 

39 Returns 

40 ------- 

41 configuration_space : ConfigurationSpace 

42 A new ConfigurationSpace object for testing. 

43 """ 

44 (kernel_sched_migration_cost_ns_special, kernel_sched_migration_cost_ns_type) = ( 

45 special_param_names("kernel_sched_migration_cost_ns") 

46 ) 

47 

48 # NOTE: FLAML requires distribution to be uniform 

49 spaces = ConfigurationSpace( 

50 space=[ 

51 CategoricalHyperparameter( 

52 name="vmSize", 

53 choices=["Standard_B2s", "Standard_B2ms", "Standard_B4ms"], 

54 default_value="Standard_B4ms", 

55 meta={"group": "provision", "cost": 0}, 

56 ), 

57 CategoricalHyperparameter( 

58 name="idle", 

59 choices=["halt", "mwait", "noidle"], 

60 default_value="halt", 

61 meta={"group": "boot", "cost": 0}, 

62 ), 

63 Integer( 

64 name="kernel_sched_latency_ns", 

65 bounds=(0, 1000000000), 

66 log=False, 

67 default=2000000, 

68 meta={ 

69 "group": "kernel", 

70 "cost": 0, 

71 QUANTIZATION_BINS_META_KEY: 11, 

72 }, 

73 ), 

74 Integer( 

75 name="kernel_sched_migration_cost_ns", 

76 bounds=(0, 500000), 

77 log=False, 

78 default=250000, 

79 meta={"group": "kernel", "cost": 0}, 

80 ), 

81 CategoricalHyperparameter( 

82 name=kernel_sched_migration_cost_ns_special, 

83 choices=[-1, 0], 

84 weights=[0.5, 0.5], 

85 default_value=-1, 

86 meta={"group": "kernel", "cost": 0}, 

87 ), 

88 CategoricalHyperparameter( 

89 name=kernel_sched_migration_cost_ns_type, 

90 choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE], 

91 weights=[0.5, 0.5], 

92 default_value=TunableValueKind.SPECIAL, 

93 ), 

94 ] 

95 ) 

96 spaces.add( 

97 [ 

98 EqualsCondition( 

99 spaces[kernel_sched_migration_cost_ns_special], 

100 spaces[kernel_sched_migration_cost_ns_type], 

101 TunableValueKind.SPECIAL, 

102 ), 

103 EqualsCondition( 

104 spaces["kernel_sched_migration_cost_ns"], 

105 spaces[kernel_sched_migration_cost_ns_type], 

106 TunableValueKind.RANGE, 

107 ), 

108 ] 

109 ) 

110 return monkey_patch_cs_quantization(spaces) 

111 

112 

113def _cmp_tunable_hyperparameter_categorical(tunable: Tunable, space: ConfigurationSpace) -> None: 

114 """Check if categorical Tunable and ConfigSpace Hyperparameter actually match.""" 

115 param = space[tunable.name] 

116 assert isinstance(param, CategoricalHyperparameter) 

117 assert set(param.choices) == set(tunable.categories) 

118 assert param.default_value == tunable.value 

119 

120 

121def _cmp_tunable_hyperparameter_numerical(tunable: Tunable, space: ConfigurationSpace) -> None: 

122 """Check if integer Tunable and ConfigSpace Hyperparameter actually match.""" 

123 param = space[tunable.name] 

124 assert isinstance(param, (UniformIntegerHyperparameter, UniformFloatHyperparameter)) 

125 assert (param.lower, param.upper) == tuple(tunable.range) 

126 if tunable.in_range(tunable.value): 

127 assert param.default_value == tunable.value 

128 assert (param.meta or {}).get(QUANTIZATION_BINS_META_KEY) == tunable.quantization_bins 

129 if tunable.quantization_bins: 

130 assert param.sample_value() in list(tunable.quantized_values or []) 

131 

132 

133def test_tunable_to_configspace_categorical(tunable_categorical: Tunable) -> None: 

134 """Check the conversion of Tunable to CategoricalHyperparameter.""" 

135 cs_param = _tunable_to_configspace(tunable_categorical) 

136 _cmp_tunable_hyperparameter_categorical(tunable_categorical, cs_param) 

137 

138 

139def test_tunable_to_configspace_int(tunable_int: Tunable) -> None: 

140 """Check the conversion of Tunable to UniformIntegerHyperparameter.""" 

141 cs_param = _tunable_to_configspace(tunable_int) 

142 _cmp_tunable_hyperparameter_numerical(tunable_int, cs_param) 

143 

144 

145def test_tunable_to_configspace_float(tunable_float: Tunable) -> None: 

146 """Check the conversion of Tunable to UniformFloatHyperparameter.""" 

147 cs_param = _tunable_to_configspace(tunable_float) 

148 _cmp_tunable_hyperparameter_numerical(tunable_float, cs_param) 

149 

150 

151_CMP_FUNC = { 

152 "int": _cmp_tunable_hyperparameter_numerical, 

153 "float": _cmp_tunable_hyperparameter_numerical, 

154 "categorical": _cmp_tunable_hyperparameter_categorical, 

155} 

156 

157 

158def test_tunable_groups_to_hyperparameters(tunable_groups: TunableGroups) -> None: 

159 """ 

160 Check the conversion of TunableGroups to ConfigurationSpace. 

161 

162 Make sure that the corresponding Tunable and Hyperparameter objects match. 

163 """ 

164 space = tunable_groups_to_configspace(tunable_groups) 

165 for tunable, _group in tunable_groups: 

166 _CMP_FUNC[tunable.type](tunable, space) 

167 

168 

169def test_tunable_groups_to_configspace( 

170 tunable_groups: TunableGroups, 

171 configuration_space: ConfigurationSpace, 

172) -> None: 

173 """Check the conversion of the entire TunableGroups collection to a single 

174 ConfigurationSpace object. 

175 """ 

176 space = tunable_groups_to_configspace(tunable_groups) 

177 assert space == configuration_space