Coverage for mlos_bench/mlos_bench/optimizers/convert_configspace.py: 98%

87 statements  

« prev     ^ index     » next       coverage.py v7.6.9, created at 2024-12-20 00:44 +0000

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Functions to convert TunableGroups to ConfigSpace for use with the mlos_core 

6optimizers. 

7""" 

8 

9import logging 

10from collections.abc import Hashable 

11from typing import Dict, List, Optional, Tuple, Union 

12 

13from ConfigSpace import ( 

14 Beta, 

15 CategoricalHyperparameter, 

16 Configuration, 

17 ConfigurationSpace, 

18 EqualsCondition, 

19 Float, 

20 Integer, 

21 Normal, 

22 Uniform, 

23) 

24from ConfigSpace.hyperparameters import NumericalHyperparameter 

25from ConfigSpace.types import NotSet 

26 

27from mlos_bench.tunables.tunable import Tunable, TunableValue 

28from mlos_bench.tunables.tunable_groups import TunableGroups 

29from mlos_bench.util import try_parse_val 

30from mlos_core.spaces.converters.util import ( 

31 QUANTIZATION_BINS_META_KEY, 

32 monkey_patch_hp_quantization, 

33) 

34 

35_LOG = logging.getLogger(__name__) 

36 

37 

38class TunableValueKind: 

39 """ 

40 Enum for the kind of the tunable value (special or not). 

41 

42 It is not a true enum because ConfigSpace wants string values. 

43 """ 

44 

45 # pylint: disable=too-few-public-methods 

46 SPECIAL = "special" 

47 RANGE = "range" 

48 

49 

50def _normalize_weights(weights: List[float]) -> List[float]: 

51 """Helper function for normalizing weights to probabilities.""" 

52 total = sum(weights) 

53 return [w / total for w in weights] 

54 

55 

56def _tunable_to_configspace( 

57 tunable: Tunable, 

58 group_name: Optional[str] = None, 

59 cost: int = 0, 

60) -> ConfigurationSpace: 

61 """ 

62 Convert a single Tunable to an equivalent set of ConfigSpace Hyperparameter objects, 

63 wrapped in a ConfigurationSpace for composability. 

64 Note: this may be more than one Hyperparameter in the case of special value handling. 

65 

66 Parameters 

67 ---------- 

68 tunable : Tunable 

69 An mlos_bench Tunable object. 

70 group_name : str 

71 Human-readable id of the CovariantTunableGroup this Tunable belongs to. 

72 cost : int 

73 Cost to change this parameter (comes from the corresponding CovariantTunableGroup). 

74 

75 Returns 

76 ------- 

77 cs : ConfigSpace.ConfigurationSpace 

78 A ConfigurationSpace object that corresponds to the Tunable. 

79 """ 

80 # pylint: disable=too-complex 

81 meta: Dict[Hashable, TunableValue] = {"cost": cost} 

82 if group_name is not None: 

83 meta["group"] = group_name 

84 if tunable.is_numerical and tunable.quantization_bins: 

85 # Temporary workaround to dropped quantization support in ConfigSpace 1.0 

86 # See Also: https://github.com/automl/ConfigSpace/issues/390 

87 meta[QUANTIZATION_BINS_META_KEY] = tunable.quantization_bins 

88 

89 if tunable.type == "categorical": 

90 return ConfigurationSpace( 

91 { 

92 tunable.name: CategoricalHyperparameter( 

93 name=tunable.name, 

94 choices=tunable.categories, 

95 weights=_normalize_weights(tunable.weights) if tunable.weights else None, 

96 default_value=tunable.default, 

97 meta=meta, 

98 ) 

99 } 

100 ) 

101 

102 distribution: Union[Uniform, Normal, Beta, None] = None 

103 if tunable.distribution == "uniform": 

104 distribution = Uniform() 

105 elif tunable.distribution == "normal": 

106 distribution = Normal( 

107 mu=tunable.distribution_params["mu"], 

108 sigma=tunable.distribution_params["sigma"], 

109 ) 

110 elif tunable.distribution == "beta": 

111 distribution = Beta( 

112 alpha=tunable.distribution_params["alpha"], 

113 beta=tunable.distribution_params["beta"], 

114 ) 

115 elif tunable.distribution is not None: 

116 raise TypeError(f"Invalid Distribution Type: {tunable.distribution}") 

117 

118 range_hp: NumericalHyperparameter 

119 if tunable.type == "int": 

120 range_hp = Integer( 

121 name=tunable.name, 

122 bounds=(int(tunable.range[0]), int(tunable.range[1])), 

123 log=bool(tunable.is_log), 

124 distribution=distribution, 

125 default=( 

126 int(tunable.default) 

127 if tunable.in_range(tunable.default) and tunable.default is not None 

128 else None 

129 ), 

130 meta=meta, 

131 ) 

132 elif tunable.type == "float": 

133 range_hp = Float( 

134 name=tunable.name, 

135 bounds=tunable.range, 

136 log=bool(tunable.is_log), 

137 distribution=distribution, 

138 default=( 

139 float(tunable.default) 

140 if tunable.in_range(tunable.default) and tunable.default is not None 

141 else None 

142 ), 

143 meta=meta, 

144 ) 

145 else: 

146 raise TypeError(f"Invalid Parameter Type: {tunable.type}") 

147 

148 monkey_patch_hp_quantization(range_hp) 

149 if not tunable.special: 

150 return ConfigurationSpace(space=[range_hp]) 

151 

152 # Compute the probabilities of switching between regular and special values. 

153 special_weights: Optional[List[float]] = None 

154 switch_weights = [0.5, 0.5] # FLAML requires uniform weights. 

155 if tunable.weights and tunable.range_weight is not None: 

156 special_weights = _normalize_weights(tunable.weights) 

157 switch_weights = _normalize_weights([sum(tunable.weights), tunable.range_weight]) 

158 

159 # Create three hyperparameters: one for regular values, 

160 # one for special values, and one to choose between the two. 

161 (special_name, type_name) = special_param_names(tunable.name) 

162 conf_space = ConfigurationSpace( 

163 space=[ 

164 range_hp, 

165 CategoricalHyperparameter( 

166 name=special_name, 

167 choices=tunable.special, 

168 weights=special_weights, 

169 default_value=tunable.default if tunable.default in tunable.special else NotSet, 

170 meta=meta, 

171 ), 

172 CategoricalHyperparameter( 

173 name=type_name, 

174 choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE], 

175 weights=switch_weights, 

176 default_value=TunableValueKind.SPECIAL, 

177 ), 

178 ] 

179 ) 

180 conf_space.add( 

181 [ 

182 EqualsCondition( 

183 conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL 

184 ), 

185 EqualsCondition( 

186 conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE 

187 ), 

188 ] 

189 ) 

190 return conf_space 

191 

192 

193def tunable_groups_to_configspace( 

194 tunables: TunableGroups, 

195 seed: Optional[int] = None, 

196) -> ConfigurationSpace: 

197 """ 

198 Convert TunableGroups to hyperparameters in ConfigurationSpace. 

199 

200 Parameters 

201 ---------- 

202 tunables : TunableGroups 

203 A collection of tunable parameters. 

204 

205 seed : Optional[int] 

206 Random seed to use. 

207 

208 Returns 

209 ------- 

210 configspace : ConfigSpace.ConfigurationSpace 

211 A new ConfigurationSpace instance that corresponds to the input TunableGroups. 

212 """ 

213 space = ConfigurationSpace(seed=seed) 

214 for tunable, group in tunables: 

215 space.add_configuration_space( 

216 prefix="", 

217 delimiter="", 

218 configuration_space=_tunable_to_configspace( 

219 tunable, 

220 group.name, 

221 group.get_current_cost(), 

222 ), 

223 ) 

224 return space 

225 

226 

227def tunable_values_to_configuration(tunables: TunableGroups) -> Configuration: 

228 """ 

229 Converts a TunableGroups current values to a ConfigSpace Configuration. 

230 

231 Parameters 

232 ---------- 

233 tunables : TunableGroups 

234 The TunableGroups to take the current value from. 

235 

236 Returns 

237 ------- 

238 ConfigSpace.Configuration 

239 A ConfigSpace Configuration. 

240 """ 

241 values: Dict[str, TunableValue] = {} 

242 for tunable, _group in tunables: 

243 if tunable.special: 

244 (special_name, type_name) = special_param_names(tunable.name) 

245 if tunable.value in tunable.special: 

246 values[type_name] = TunableValueKind.SPECIAL 

247 values[special_name] = tunable.value 

248 else: 

249 values[type_name] = TunableValueKind.RANGE 

250 values[tunable.name] = tunable.value 

251 else: 

252 values[tunable.name] = tunable.value 

253 configspace = tunable_groups_to_configspace(tunables) 

254 return Configuration(configspace, values=values) 

255 

256 

257def configspace_data_to_tunable_values(data: dict) -> Dict[str, TunableValue]: 

258 """ 

259 Remove the fields that correspond to special values in ConfigSpace. 

260 

261 In particular, remove and keys suffixes added by `special_param_names`. 

262 """ 

263 data = data.copy() 

264 specials = [special_param_name_strip(k) for k in data.keys() if special_param_name_is_temp(k)] 

265 for k in specials: 

266 (special_name, type_name) = special_param_names(k) 

267 if data[type_name] == TunableValueKind.SPECIAL: 

268 data[k] = data[special_name] 

269 if special_name in data: 

270 del data[special_name] 

271 del data[type_name] 

272 # May need to convert numpy values to regular types. 

273 data = {k: try_parse_val(v) for k, v in data.items()} 

274 return data 

275 

276 

277def special_param_names(name: str) -> Tuple[str, str]: 

278 """ 

279 Generate the names of the auxiliary hyperparameters that correspond to a tunable 

280 that can have special values. 

281 

282 NOTE: `!` characters are currently disallowed in Tunable names in order handle this logic. 

283 

284 Parameters 

285 ---------- 

286 name : str 

287 The name of the tunable parameter. 

288 

289 Returns 

290 ------- 

291 special_name : str 

292 The name of the hyperparameter that corresponds to the special value. 

293 type_name : str 

294 The name of the hyperparameter that chooses between the regular and the special values. 

295 """ 

296 return (name + "!special", name + "!type") 

297 

298 

299def special_param_name_is_temp(name: str) -> bool: 

300 """ 

301 Check if name corresponds to a temporary ConfigSpace parameter. 

302 

303 NOTE: `!` characters are currently disallowed in Tunable names in order handle this logic. 

304 

305 Parameters 

306 ---------- 

307 name : str 

308 The name of the hyperparameter. 

309 

310 Returns 

311 ------- 

312 is_special : bool 

313 True if the name corresponds to a temporary ConfigSpace hyperparameter. 

314 """ 

315 return name.endswith("!type") 

316 

317 

318def special_param_name_strip(name: str) -> str: 

319 """ 

320 Remove the temporary suffix from a special parameter name. 

321 

322 NOTE: `!` characters are currently disallowed in Tunable names in order handle this logic. 

323 

324 Parameters 

325 ---------- 

326 name : str 

327 The name of the hyperparameter. 

328 

329 Returns 

330 ------- 

331 stripped_name : str 

332 The name of the hyperparameter without the temporary suffix. 

333 """ 

334 return name.split("!", 1)[0]