Coverage for mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py: 99%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Tests for LlamaTune space adapter. 

7""" 

8 

9# pylint: disable=missing-function-docstring 

10 

11from typing import Any, Dict, Iterator, List, Set 

12 

13import pytest 

14 

15import ConfigSpace as CS 

16import pandas as pd 

17 

18from mlos_core.spaces.adapters import LlamaTuneAdapter 

19 

20 

21def construct_parameter_space( 

22 n_continuous_params: int = 0, 

23 n_integer_params: int = 0, 

24 n_categorical_params: int = 0, 

25 seed: int = 1234, 

26) -> CS.ConfigurationSpace: 

27 """ 

28 Helper function for construct an instance of `ConfigSpace.ConfigurationSpace`. 

29 """ 

30 input_space = CS.ConfigurationSpace(seed=seed) 

31 

32 for idx in range(n_continuous_params): 

33 input_space.add_hyperparameter( 

34 CS.UniformFloatHyperparameter(name=f'cont_{idx}', lower=0, upper=64)) 

35 for idx in range(n_integer_params): 

36 input_space.add_hyperparameter( 

37 CS.UniformIntegerHyperparameter(name=f'int_{idx}', lower=-1, upper=256)) 

38 for idx in range(n_categorical_params): 

39 input_space.add_hyperparameter( 

40 CS.CategoricalHyperparameter(name=f'str_{idx}', choices=[f'option_{idx}' for idx in range(5)])) 

41 

42 return input_space 

43 

44 

45@pytest.mark.parametrize(('num_target_space_dims', 'param_space_kwargs'), ([ 

46 (num_target_space_dims, param_space_kwargs) 

47 for num_target_space_dims in (2, 4) 

48 for num_orig_space_factor in (1.5, 4) 

49 for param_space_kwargs in ( 

50 {'n_continuous_params': int(num_target_space_dims * num_orig_space_factor)}, 

51 {'n_integer_params': int(num_target_space_dims * num_orig_space_factor)}, 

52 {'n_categorical_params': int(num_target_space_dims * num_orig_space_factor)}, 

53 # Mix of all three types 

54 { 

55 'n_continuous_params': int(num_target_space_dims * num_orig_space_factor / 3), 

56 'n_integer_params': int(num_target_space_dims * num_orig_space_factor / 3), 

57 'n_categorical_params': int(num_target_space_dims * num_orig_space_factor / 3), 

58 }, 

59 ) 

60])) 

61def test_num_low_dims(num_target_space_dims: int, param_space_kwargs: dict) -> None: # pylint: disable=too-many-locals 

62 """ 

63 Tests LlamaTune's low-to-high space projection method. 

64 """ 

65 input_space = construct_parameter_space(**param_space_kwargs) 

66 

67 # Number of target parameter space dimensions should be fewer than those of the original space 

68 with pytest.raises(ValueError): 

69 LlamaTuneAdapter( 

70 orig_parameter_space=input_space, 

71 num_low_dims=len(list(input_space.keys())) 

72 ) 

73 

74 # Enable only low-dimensional space projections 

75 adapter = LlamaTuneAdapter( 

76 orig_parameter_space=input_space, 

77 num_low_dims=num_target_space_dims, 

78 special_param_values=None, 

79 max_unique_values_per_param=None 

80 ) 

81 

82 sampled_configs = adapter.target_parameter_space.sample_configuration(size=100) 

83 for sampled_config in sampled_configs: # pylint: disable=not-an-iterable # (false positive) 

84 # Transform low-dim config to high-dim point/config 

85 sampled_config_df = pd.DataFrame([sampled_config.values()], columns=list(sampled_config.keys())) 

86 orig_config_df = adapter.transform(sampled_config_df) 

87 

88 # High-dim (i.e., original) config should be valid 

89 orig_config = CS.Configuration(input_space, values=orig_config_df.iloc[0].to_dict()) 

90 input_space.check_configuration(orig_config) 

91 

92 # Transform high-dim config back to low-dim 

93 target_config_df = adapter.inverse_transform(orig_config_df) 

94 

95 # Sampled config and this should be the same 

96 target_config = CS.Configuration(adapter.target_parameter_space, values=target_config_df.iloc[0].to_dict()) 

97 assert target_config == sampled_config 

98 

99 # Try inverse projection (i.e., high-to-low) for previously unseen configs 

100 unseen_sampled_configs = adapter.target_parameter_space.sample_configuration(size=25) 

101 for unseen_sampled_config in unseen_sampled_configs: # pylint: disable=not-an-iterable # (false positive) 

102 if unseen_sampled_config in sampled_configs: # pylint: disable=unsupported-membership-test # (false positive) 

103 continue 

104 

105 unseen_sampled_config_df = pd.DataFrame([unseen_sampled_config.values()], columns=list(unseen_sampled_config.keys())) 

106 with pytest.raises(ValueError): 

107 _ = adapter.inverse_transform(unseen_sampled_config_df) # pylint: disable=redefined-variable-type 

108 

109 

110def test_special_parameter_values_validation() -> None: 

111 """ 

112 Tests LlamaTune's validation process of user-provided special parameter values dictionary. 

113 """ 

114 input_space = CS.ConfigurationSpace(seed=1234) 

115 input_space.add_hyperparameter( 

116 CS.CategoricalHyperparameter(name='str', choices=[f'choice_{idx}' for idx in range(5)])) 

117 input_space.add_hyperparameter( 

118 CS.UniformFloatHyperparameter(name='cont', lower=-1, upper=100)) 

119 input_space.add_hyperparameter( 

120 CS.UniformIntegerHyperparameter(name='int', lower=0, upper=100)) 

121 

122 # Only UniformIntegerHyperparameters are currently supported 

123 with pytest.raises(NotImplementedError): 

124 special_param_values_dict_1 = {'str': 'choice_1'} 

125 LlamaTuneAdapter( 

126 orig_parameter_space=input_space, 

127 num_low_dims=2, 

128 special_param_values=special_param_values_dict_1, 

129 max_unique_values_per_param=None, 

130 ) 

131 

132 with pytest.raises(NotImplementedError): 

133 special_param_values_dict_2 = {'cont': -1} 

134 LlamaTuneAdapter( 

135 orig_parameter_space=input_space, 

136 num_low_dims=2, 

137 special_param_values=special_param_values_dict_2, 

138 max_unique_values_per_param=None, 

139 ) 

140 

141 # Special value should belong to parameter value domain 

142 with pytest.raises(ValueError, match='value domain'): 

143 special_param_values_dict = {'int': -1} 

144 LlamaTuneAdapter( 

145 orig_parameter_space=input_space, 

146 num_low_dims=2, 

147 special_param_values=special_param_values_dict, 

148 max_unique_values_per_param=None, 

149 ) 

150 

151 # Invalid dicts; ValueError should be thrown 

152 invalid_special_param_values_dicts: List[Dict[str, Any]] = [ 

153 {'int-Q': 0}, # parameter does not exist 

154 {'int': {0: 0.2}}, # invalid definition 

155 {'int': 0.2}, # invalid parameter value 

156 {'int': (0.4, 0)}, # (biasing %, special value) instead of (special value, biasing %) 

157 {'int': [0, 0]}, # duplicate special values 

158 {'int': []}, # empty list 

159 {'int': [{0: 0.2}]}, 

160 {'int': [(0.4, 0), (1, 0.7)]}, # first tuple is inverted; second is correct 

161 {'int': [(0, 0.1), (0, 0.2)]}, # duplicate special values 

162 ] 

163 for spv_dict in invalid_special_param_values_dicts: 

164 with pytest.raises(ValueError): 

165 LlamaTuneAdapter( 

166 orig_parameter_space=input_space, 

167 num_low_dims=2, 

168 special_param_values=spv_dict, 

169 max_unique_values_per_param=None, 

170 ) 

171 

172 # Biasing percentage of special value(s) are invalid 

173 invalid_special_param_values_dicts = [ 

174 {'int': (0, 1.1)}, # >1 probability 

175 {'int': (0, 0)}, # Zero probability 

176 {'int': (0, -0.1)}, # Negative probability 

177 {'int': (0, 20)}, # 2,000% instead of 20% 

178 {'int': [0, 1, 2, 3, 4, 5]}, # default biasing is 20%; 6 values * 20% > 100% 

179 {'int': [(0, 0.4), (1, 0.7)]}, # combined probability >100% 

180 {'int': [(0, -0.4), (1, 0.7)]}, # probability for value 0 is invalid. 

181 ] 

182 

183 for spv_dict in invalid_special_param_values_dicts: 

184 with pytest.raises(ValueError): 

185 LlamaTuneAdapter( 

186 orig_parameter_space=input_space, 

187 num_low_dims=2, 

188 special_param_values=spv_dict, 

189 max_unique_values_per_param=None, 

190 ) 

191 

192 

193def gen_random_configs(adapter: LlamaTuneAdapter, num_configs: int) -> Iterator[CS.Configuration]: 

194 for sampled_config in adapter.target_parameter_space.sample_configuration(size=num_configs): 

195 # Transform low-dim config to high-dim config 

196 sampled_config_df = pd.DataFrame([sampled_config.values()], columns=list(sampled_config.keys())) 

197 orig_config_df = adapter.transform(sampled_config_df) 

198 orig_config = CS.Configuration(adapter.orig_parameter_space, values=orig_config_df.iloc[0].to_dict()) 

199 yield orig_config 

200 

201 

202def test_special_parameter_values_biasing() -> None: # pylint: disable=too-complex 

203 """ 

204 Tests LlamaTune's special parameter values biasing methodology 

205 """ 

206 input_space = CS.ConfigurationSpace(seed=1234) 

207 input_space.add_hyperparameter( 

208 CS.UniformIntegerHyperparameter(name='int_1', lower=0, upper=100)) 

209 input_space.add_hyperparameter( 

210 CS.UniformIntegerHyperparameter(name='int_2', lower=0, upper=100)) 

211 

212 num_configs = 400 

213 bias_percentage = LlamaTuneAdapter.DEFAULT_SPECIAL_PARAM_VALUE_BIASING_PERCENTAGE 

214 eps = 0.2 

215 

216 # Single parameter; single special value 

217 special_param_value_dicts: List[Dict[str, Any]] = [ 

218 {'int_1': 0}, 

219 {'int_1': (0, bias_percentage)}, 

220 {'int_1': [0]}, 

221 {'int_1': [(0, bias_percentage)]} 

222 ] 

223 

224 for spv_dict in special_param_value_dicts: 

225 adapter = LlamaTuneAdapter( 

226 orig_parameter_space=input_space, 

227 num_low_dims=1, 

228 special_param_values=spv_dict, 

229 max_unique_values_per_param=None, 

230 ) 

231 

232 special_value_occurrences = sum( 

233 1 for config in gen_random_configs(adapter, num_configs) if config['int_1'] == 0) 

234 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences 

235 

236 # Single parameter; multiple special values 

237 special_param_value_dicts = [ 

238 {'int_1': [0, 1]}, 

239 {'int_1': [(0, bias_percentage), (1, bias_percentage)]} 

240 ] 

241 

242 for spv_dict in special_param_value_dicts: 

243 adapter = LlamaTuneAdapter( 

244 orig_parameter_space=input_space, 

245 num_low_dims=1, 

246 special_param_values=spv_dict, 

247 max_unique_values_per_param=None, 

248 ) 

249 

250 special_values_occurrences = {0: 0, 1: 0} 

251 for config in gen_random_configs(adapter, num_configs): 

252 if config['int_1'] == 0: 

253 special_values_occurrences[0] += 1 

254 elif config['int_1'] == 1: 

255 special_values_occurrences[1] += 1 

256 

257 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[0] 

258 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[1] 

259 

260 # Multiple parameters; multiple special values; different biasing percentage 

261 spv_dict = { 

262 'int_1': [(0, bias_percentage), (1, bias_percentage / 2)], 

263 'int_2': [(2, bias_percentage / 2), (100, bias_percentage * 1.5)] 

264 } 

265 adapter = LlamaTuneAdapter( 

266 orig_parameter_space=input_space, 

267 num_low_dims=1, 

268 special_param_values=spv_dict, 

269 max_unique_values_per_param=None, 

270 ) 

271 

272 special_values_instances: Dict[str, Dict[int, int]] = { 

273 'int_1': {0: 0, 1: 0}, 

274 'int_2': {2: 0, 100: 0}, 

275 } 

276 for config in gen_random_configs(adapter, num_configs): 

277 if config['int_1'] == 0: 

278 special_values_instances['int_1'][0] += 1 

279 elif config['int_1'] == 1: 

280 special_values_instances['int_1'][1] += 1 

281 

282 if config['int_2'] == 2: 

283 special_values_instances['int_2'][2] += 1 

284 elif config['int_2'] == 100: 

285 special_values_instances['int_2'][100] += 1 

286 

287 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_instances['int_1'][0] 

288 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= special_values_instances['int_1'][1] 

289 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= special_values_instances['int_2'][2] 

290 assert (1 - eps) * int(num_configs * bias_percentage * 1.5) <= special_values_instances['int_2'][100] 

291 

292 

293def test_max_unique_values_per_param() -> None: 

294 """ 

295 Tests LlamaTune's parameter values discretization implementation. 

296 """ 

297 # Define config space with a mix of different parameter types 

298 input_space = CS.ConfigurationSpace(seed=1234) 

299 input_space.add_hyperparameter( 

300 CS.UniformFloatHyperparameter(name='cont_1', lower=0, upper=5)) 

301 input_space.add_hyperparameter( 

302 CS.UniformFloatHyperparameter(name='cont_2', lower=1, upper=100)) 

303 input_space.add_hyperparameter( 

304 CS.UniformIntegerHyperparameter(name='int_1', lower=1, upper=10)) 

305 input_space.add_hyperparameter( 

306 CS.UniformIntegerHyperparameter(name='int_2', lower=0, upper=2048)) 

307 input_space.add_hyperparameter( 

308 CS.CategoricalHyperparameter(name='str_1', choices=['on', 'off'])) 

309 input_space.add_hyperparameter( 

310 CS.CategoricalHyperparameter(name='str_2', choices=[f'choice_{idx}' for idx in range(10)])) 

311 

312 # Restrict the number of unique parameter values 

313 num_configs = 200 

314 for max_unique_values_per_param in (5, 25, 100): 

315 adapter = LlamaTuneAdapter( 

316 orig_parameter_space=input_space, 

317 num_low_dims=3, 

318 special_param_values=None, 

319 max_unique_values_per_param=max_unique_values_per_param, 

320 ) 

321 

322 # Keep track of unique values generated for each parameter 

323 unique_values_dict: Dict[str, set] = {param: set() for param in list(input_space.keys())} 

324 for config in gen_random_configs(adapter, num_configs): 

325 for param, value in config.items(): 

326 unique_values_dict[param].add(value) 

327 

328 # Ensure that their number is less than the maximum number allowed 

329 for _, unique_values in unique_values_dict.items(): 

330 assert len(unique_values) <= max_unique_values_per_param 

331 

332 

333@pytest.mark.parametrize(('num_target_space_dims', 'param_space_kwargs'), ([ 

334 (num_target_space_dims, param_space_kwargs) 

335 for num_target_space_dims in (2, 4) 

336 for num_orig_space_factor in (1.5, 4) 

337 for param_space_kwargs in ( 

338 {'n_continuous_params': int(num_target_space_dims * num_orig_space_factor)}, 

339 {'n_integer_params': int(num_target_space_dims * num_orig_space_factor)}, 

340 {'n_categorical_params': int(num_target_space_dims * num_orig_space_factor)}, 

341 # Mix of all three types 

342 { 

343 'n_continuous_params': int(num_target_space_dims * num_orig_space_factor / 3), 

344 'n_integer_params': int(num_target_space_dims * num_orig_space_factor / 3), 

345 'n_categorical_params': int(num_target_space_dims * num_orig_space_factor / 3), 

346 }, 

347 ) 

348])) 

349def test_approx_inverse_mapping(num_target_space_dims: int, param_space_kwargs: dict) -> None: # pylint: disable=too-many-locals 

350 """ 

351 Tests LlamaTune's approximate high-to-low space projection method, using pseudo-inverse. 

352 """ 

353 input_space = construct_parameter_space(**param_space_kwargs) 

354 

355 # Enable low-dimensional space projection, but disable reverse mapping 

356 adapter = LlamaTuneAdapter( 

357 orig_parameter_space=input_space, 

358 num_low_dims=num_target_space_dims, 

359 special_param_values=None, 

360 max_unique_values_per_param=None, 

361 use_approximate_reverse_mapping=False, 

362 ) 

363 

364 sampled_config = input_space.sample_configuration() # size=1) 

365 with pytest.raises(ValueError): 

366 sampled_config_df = pd.DataFrame([sampled_config.values()], columns=list(sampled_config.keys())) 

367 _ = adapter.inverse_transform(sampled_config_df) 

368 

369 # Enable low-dimensional space projection *and* reverse mapping 

370 adapter = LlamaTuneAdapter( 

371 orig_parameter_space=input_space, 

372 num_low_dims=num_target_space_dims, 

373 special_param_values=None, 

374 max_unique_values_per_param=None, 

375 use_approximate_reverse_mapping=True, 

376 ) 

377 

378 # Warning should be printed the first time 

379 sampled_config = input_space.sample_configuration() # size=1) 

380 with pytest.warns(UserWarning): 

381 sampled_config_df = pd.DataFrame([sampled_config.values()], columns=list(sampled_config.keys())) 

382 target_config_df = adapter.inverse_transform(sampled_config_df) 

383 # Low-dim (i.e., target) config should be valid 

384 target_config = CS.Configuration(adapter.target_parameter_space, values=target_config_df.iloc[0].to_dict()) 

385 adapter.target_parameter_space.check_configuration(target_config) 

386 

387 # Test inverse transform with 100 random configs 

388 for _ in range(100): 

389 sampled_config = input_space.sample_configuration() # size=1) 

390 sampled_config_df = pd.DataFrame([sampled_config.values()], columns=list(sampled_config.keys())) 

391 target_config_df = adapter.inverse_transform(sampled_config_df) 

392 # Low-dim (i.e., target) config should be valid 

393 target_config = CS.Configuration(adapter.target_parameter_space, values=target_config_df.iloc[0].to_dict()) 

394 adapter.target_parameter_space.check_configuration(target_config) 

395 

396 

397@pytest.mark.parametrize(('num_low_dims', 'special_param_values', 'max_unique_values_per_param'), ([ 

398 (num_low_dims, special_param_values, max_unique_values_per_param) 

399 for num_low_dims in (8, 16) 

400 for special_param_values in ( 

401 {'int_1': -1, 'int_2': -1, 'int_3': -1, 'int_4': [-1, 0]}, 

402 {'int_1': (-1, 0.1), 'int_2': -1, 'int_3': (-1, 0.3), 'int_4': [(-1, 0.1), (0, 0.2)]}, 

403 ) 

404 for max_unique_values_per_param in (50, 250) 

405])) 

406def test_llamatune_pipeline(num_low_dims: int, special_param_values: dict, max_unique_values_per_param: int) -> None: 

407 """ 

408 Tests LlamaTune space adapter when all components are active. 

409 """ 

410 # pylint: disable=too-many-locals 

411 

412 # Define config space with a mix of different parameter types 

413 input_space = construct_parameter_space(n_continuous_params=10, n_integer_params=10, n_categorical_params=5) 

414 adapter = LlamaTuneAdapter( 

415 orig_parameter_space=input_space, 

416 num_low_dims=num_low_dims, 

417 special_param_values=special_param_values, 

418 max_unique_values_per_param=max_unique_values_per_param, 

419 ) 

420 

421 special_value_occurrences = { 

422 param: {special_value: 0 for special_value, _ in tuples_list} 

423 for param, tuples_list in adapter._special_param_values_dict.items() # pylint: disable=protected-access 

424 } 

425 unique_values_dict: Dict[str, Set] = {param: set() for param in input_space.keys()} 

426 

427 num_configs = 1000 

428 for config in adapter.target_parameter_space.sample_configuration(size=num_configs): # pylint: disable=not-an-iterable 

429 # Transform low-dim config to high-dim point/config 

430 sampled_config_df = pd.DataFrame([config.values()], columns=list(config.keys())) 

431 orig_config_df = adapter.transform(sampled_config_df) 

432 # High-dim (i.e., original) config should be valid 

433 orig_config = CS.Configuration(input_space, values=orig_config_df.iloc[0].to_dict()) 

434 input_space.check_configuration(orig_config) 

435 

436 # Transform high-dim config back to low-dim 

437 target_config_df = adapter.inverse_transform(orig_config_df) 

438 # Sampled config and this should be the same 

439 target_config = CS.Configuration(adapter.target_parameter_space, values=target_config_df.iloc[0].to_dict()) 

440 assert target_config == config 

441 

442 for param, value in orig_config.items(): 

443 # Keep track of special value occurrences 

444 if param in special_value_occurrences: 

445 if value in special_value_occurrences[param]: 

446 special_value_occurrences[param][value] += 1 

447 

448 # Keep track of unique values generated for each parameter 

449 unique_values_dict[param].add(value) 

450 

451 # Ensure that occurrences of special values do not significantly deviate from expected 

452 eps = 0.2 

453 for param, tuples_list in adapter._special_param_values_dict.items(): # pylint: disable=protected-access 

454 for value, bias_percentage in tuples_list: 

455 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences[param][value] 

456 

457 # Ensure that number of unique values is less than the maximum number allowed 

458 for _, unique_values in unique_values_dict.items(): 

459 assert len(unique_values) <= max_unique_values_per_param 

460 

461 

462@pytest.mark.parametrize(('num_target_space_dims', 'param_space_kwargs'), ([ 

463 (num_target_space_dims, param_space_kwargs) 

464 for num_target_space_dims in (2, 4) 

465 for num_orig_space_factor in (1.5, 4) 

466 for param_space_kwargs in ( 

467 {'n_continuous_params': int(num_target_space_dims * num_orig_space_factor)}, 

468 {'n_integer_params': int(num_target_space_dims * num_orig_space_factor)}, 

469 {'n_categorical_params': int(num_target_space_dims * num_orig_space_factor)}, 

470 # Mix of all three types 

471 { 

472 'n_continuous_params': int(num_target_space_dims * num_orig_space_factor / 3), 

473 'n_integer_params': int(num_target_space_dims * num_orig_space_factor / 3), 

474 'n_categorical_params': int(num_target_space_dims * num_orig_space_factor / 3), 

475 }, 

476 ) 

477])) 

478def test_deterministic_behavior_for_same_seed(num_target_space_dims: int, param_space_kwargs: dict) -> None: 

479 """ 

480 Tests LlamaTune's space adapter deterministic behavior when given same seed in the input parameter space. 

481 """ 

482 def generate_target_param_space_configs(seed: int) -> List[CS.Configuration]: 

483 input_space = construct_parameter_space(**param_space_kwargs, seed=seed) 

484 

485 # Init adapter and sample points in the low-dim space 

486 adapter = LlamaTuneAdapter( 

487 orig_parameter_space=input_space, 

488 num_low_dims=num_target_space_dims, 

489 special_param_values=None, 

490 max_unique_values_per_param=None, 

491 use_approximate_reverse_mapping=False, 

492 ) 

493 

494 sample_configs: List[CS.Configuration] = adapter.target_parameter_space.sample_configuration(size=100) 

495 return sample_configs 

496 

497 assert generate_target_param_space_configs(42) == generate_target_param_space_configs(42) 

498 assert generate_target_param_space_configs(1234) != generate_target_param_space_configs(42)