Coverage for mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py: 99%
171 statements
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-20 00:44 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Tests for LlamaTune space adapter."""
7# pylint: disable=missing-function-docstring
9from typing import Any, Dict, Iterator, List, Set
11import ConfigSpace as CS
12import pandas as pd
13import pytest
15from mlos_core.spaces.adapters import LlamaTuneAdapter
16from mlos_core.spaces.converters.util import (
17 QUANTIZATION_BINS_META_KEY,
18 monkey_patch_cs_quantization,
19)
21# Explicitly test quantized values with llamatune space adapter.
22# TODO: Add log scale sampling tests as well.
25def construct_parameter_space( # pylint: disable=too-many-arguments
26 *,
27 n_continuous_params: int = 0,
28 n_quantized_continuous_params: int = 0,
29 n_integer_params: int = 0,
30 n_quantized_integer_params: int = 0,
31 n_categorical_params: int = 0,
32 seed: int = 1234,
33) -> CS.ConfigurationSpace:
34 """Helper function for construct an instance of `ConfigSpace.ConfigurationSpace`."""
35 input_space = CS.ConfigurationSpace(
36 seed=seed,
37 space=[
38 *(
39 CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64)
40 for idx in range(n_continuous_params)
41 ),
42 *(
43 CS.UniformFloatHyperparameter(
44 name=f"cont_{idx}", lower=0, upper=64, meta={QUANTIZATION_BINS_META_KEY: 6}
45 )
46 for idx in range(n_quantized_continuous_params)
47 ),
48 *(
49 CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256)
50 for idx in range(n_integer_params)
51 ),
52 *(
53 CS.UniformIntegerHyperparameter(
54 name=f"int_{idx}", lower=0, upper=256, meta={QUANTIZATION_BINS_META_KEY: 17}
55 )
56 for idx in range(n_quantized_integer_params)
57 ),
58 *(
59 CS.CategoricalHyperparameter(
60 name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)]
61 )
62 for idx in range(n_categorical_params)
63 ),
64 ],
65 )
66 return monkey_patch_cs_quantization(input_space)
69@pytest.mark.parametrize(
70 ("num_target_space_dims", "param_space_kwargs"),
71 (
72 [
73 (num_target_space_dims, param_space_kwargs)
74 for num_target_space_dims in (2, 4)
75 for num_orig_space_factor in (1.5, 4)
76 for param_space_kwargs in (
77 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
78 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
79 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
80 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
81 {"n_quantized_integer_params": int(num_target_space_dims * num_orig_space_factor)},
82 {
83 "n_quantized_continuous_params": int(
84 num_target_space_dims * num_orig_space_factor
85 )
86 },
87 # Mix of all three types
88 {
89 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
90 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
91 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
92 },
93 )
94 ]
95 ),
96)
97def test_num_low_dims(
98 num_target_space_dims: int,
99 param_space_kwargs: dict,
100) -> None: # pylint: disable=too-many-locals
101 """Tests LlamaTune's low-to-high space projection method."""
102 input_space = construct_parameter_space(**param_space_kwargs)
104 # Number of target parameter space dimensions should be fewer than those of the original space
105 with pytest.raises(ValueError):
106 LlamaTuneAdapter(
107 orig_parameter_space=input_space, num_low_dims=len(list(input_space.keys()))
108 )
110 # Enable only low-dimensional space projections
111 adapter = LlamaTuneAdapter(
112 orig_parameter_space=input_space,
113 num_low_dims=num_target_space_dims,
114 special_param_values=None,
115 max_unique_values_per_param=None,
116 )
118 sampled_configs = adapter.target_parameter_space.sample_configuration(size=100)
119 for sampled_config in sampled_configs: # pylint: disable=not-an-iterable # (false positive)
120 # Transform low-dim config to high-dim point/config
121 sampled_config_sr = pd.Series(dict(sampled_config))
122 orig_config_sr = adapter.transform(sampled_config_sr)
124 # High-dim (i.e., original) config should be valid
125 orig_config = CS.Configuration(input_space, values=orig_config_sr.to_dict())
126 orig_config.check_valid_configuration()
128 # Transform high-dim config back to low-dim
129 target_config_sr = adapter.inverse_transform(orig_config_sr)
131 # Sampled config and this should be the same
132 target_config = CS.Configuration(
133 adapter.target_parameter_space,
134 values=target_config_sr.to_dict(),
135 )
136 assert target_config == sampled_config
138 # Try inverse projection (i.e., high-to-low) for previously unseen configs
139 unseen_sampled_configs = adapter.target_parameter_space.sample_configuration(size=25)
140 for (
141 unseen_sampled_config
142 ) in unseen_sampled_configs: # pylint: disable=not-an-iterable # (false positive)
143 if (
144 unseen_sampled_config
145 in sampled_configs # pylint: disable=unsupported-membership-test # (false positive)
146 ):
147 continue
149 unseen_sampled_config_sr = pd.Series(dict(unseen_sampled_config))
150 with pytest.raises(ValueError):
151 _ = adapter.inverse_transform(
152 unseen_sampled_config_sr
153 ) # pylint: disable=redefined-variable-type
156def test_special_parameter_values_validation() -> None:
157 """Tests LlamaTune's validation process of user-provided special parameter values
158 dictionary.
159 """
160 input_space = CS.ConfigurationSpace(seed=1234)
161 input_space.add(
162 CS.CategoricalHyperparameter(name="str", choices=[f"choice_{idx}" for idx in range(5)])
163 )
164 input_space.add(CS.UniformFloatHyperparameter(name="cont", lower=-1, upper=100))
165 input_space.add(CS.UniformIntegerHyperparameter(name="int", lower=0, upper=100))
167 # Only UniformIntegerHyperparameters are currently supported
168 with pytest.raises(NotImplementedError):
169 special_param_values_dict_1 = {"str": "choice_1"}
170 LlamaTuneAdapter(
171 orig_parameter_space=input_space,
172 num_low_dims=2,
173 special_param_values=special_param_values_dict_1,
174 max_unique_values_per_param=None,
175 )
177 with pytest.raises(NotImplementedError):
178 special_param_values_dict_2 = {"cont": -1}
179 LlamaTuneAdapter(
180 orig_parameter_space=input_space,
181 num_low_dims=2,
182 special_param_values=special_param_values_dict_2,
183 max_unique_values_per_param=None,
184 )
186 # Special value should belong to parameter value domain
187 with pytest.raises(ValueError, match="value domain"):
188 special_param_values_dict = {"int": -1}
189 LlamaTuneAdapter(
190 orig_parameter_space=input_space,
191 num_low_dims=2,
192 special_param_values=special_param_values_dict,
193 max_unique_values_per_param=None,
194 )
196 # Invalid dicts; ValueError should be thrown
197 invalid_special_param_values_dicts: List[Dict[str, Any]] = [
198 {"int-Q": 0}, # parameter does not exist
199 {"int": {0: 0.2}}, # invalid definition
200 {"int": 0.2}, # invalid parameter value
201 {"int": (0.4, 0)}, # (biasing %, special value) instead of (special value, biasing %)
202 {"int": [0, 0]}, # duplicate special values
203 {"int": []}, # empty list
204 {"int": [{0: 0.2}]},
205 {"int": [(0.4, 0), (1, 0.7)]}, # first tuple is inverted; second is correct
206 {"int": [(0, 0.1), (0, 0.2)]}, # duplicate special values
207 ]
208 for spv_dict in invalid_special_param_values_dicts:
209 with pytest.raises(ValueError):
210 LlamaTuneAdapter(
211 orig_parameter_space=input_space,
212 num_low_dims=2,
213 special_param_values=spv_dict,
214 max_unique_values_per_param=None,
215 )
217 # Biasing percentage of special value(s) are invalid
218 invalid_special_param_values_dicts = [
219 {"int": (0, 1.1)}, # >1 probability
220 {"int": (0, 0)}, # Zero probability
221 {"int": (0, -0.1)}, # Negative probability
222 {"int": (0, 20)}, # 2,000% instead of 20%
223 {"int": [0, 1, 2, 3, 4, 5]}, # default biasing is 20%; 6 values * 20% > 100%
224 {"int": [(0, 0.4), (1, 0.7)]}, # combined probability >100%
225 {"int": [(0, -0.4), (1, 0.7)]}, # probability for value 0 is invalid.
226 ]
228 for spv_dict in invalid_special_param_values_dicts:
229 with pytest.raises(ValueError):
230 LlamaTuneAdapter(
231 orig_parameter_space=input_space,
232 num_low_dims=2,
233 special_param_values=spv_dict,
234 max_unique_values_per_param=None,
235 )
238def gen_random_configs(adapter: LlamaTuneAdapter, num_configs: int) -> Iterator[CS.Configuration]:
239 for sampled_config in adapter.target_parameter_space.sample_configuration(size=num_configs):
240 # Transform low-dim config to high-dim config
241 sampled_config_sr = pd.Series(dict(sampled_config))
242 orig_config_sr = adapter.transform(sampled_config_sr)
243 orig_config = CS.Configuration(
244 adapter.orig_parameter_space,
245 values=orig_config_sr.to_dict(),
246 )
247 yield orig_config
250def test_special_parameter_values_biasing() -> None: # pylint: disable=too-complex
251 """Tests LlamaTune's special parameter values biasing methodology."""
252 input_space = CS.ConfigurationSpace(seed=1234)
253 input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100))
254 input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=100))
256 num_configs = 400
257 bias_percentage = LlamaTuneAdapter.DEFAULT_SPECIAL_PARAM_VALUE_BIASING_PERCENTAGE
258 eps = 0.2
260 # Single parameter; single special value
261 special_param_value_dicts: List[Dict[str, Any]] = [
262 {"int_1": 0},
263 {"int_1": (0, bias_percentage)},
264 {"int_1": [0]},
265 {"int_1": [(0, bias_percentage)]},
266 ]
268 for spv_dict in special_param_value_dicts:
269 adapter = LlamaTuneAdapter(
270 orig_parameter_space=input_space,
271 num_low_dims=1,
272 special_param_values=spv_dict,
273 max_unique_values_per_param=None,
274 )
276 special_value_occurrences = sum(
277 1 for config in gen_random_configs(adapter, num_configs) if config["int_1"] == 0
278 )
279 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences
281 # Single parameter; multiple special values
282 special_param_value_dicts = [
283 {"int_1": [0, 1]},
284 {"int_1": [(0, bias_percentage), (1, bias_percentage)]},
285 ]
287 for spv_dict in special_param_value_dicts:
288 adapter = LlamaTuneAdapter(
289 orig_parameter_space=input_space,
290 num_low_dims=1,
291 special_param_values=spv_dict,
292 max_unique_values_per_param=None,
293 )
295 special_values_occurrences = {0: 0, 1: 0}
296 for config in gen_random_configs(adapter, num_configs):
297 if config["int_1"] == 0:
298 special_values_occurrences[0] += 1
299 elif config["int_1"] == 1:
300 special_values_occurrences[1] += 1
302 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[0]
303 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[1]
305 # Multiple parameters; multiple special values; different biasing percentage
306 spv_dict = {
307 "int_1": [(0, bias_percentage), (1, bias_percentage / 2)],
308 "int_2": [(2, bias_percentage / 2), (100, bias_percentage * 1.5)],
309 }
310 adapter = LlamaTuneAdapter(
311 orig_parameter_space=input_space,
312 num_low_dims=1,
313 special_param_values=spv_dict,
314 max_unique_values_per_param=None,
315 )
317 special_values_instances: Dict[str, Dict[int, int]] = {
318 "int_1": {0: 0, 1: 0},
319 "int_2": {2: 0, 100: 0},
320 }
321 for config in gen_random_configs(adapter, num_configs):
322 if config["int_1"] == 0:
323 special_values_instances["int_1"][0] += 1
324 elif config["int_1"] == 1:
325 special_values_instances["int_1"][1] += 1
327 if config["int_2"] == 2:
328 special_values_instances["int_2"][2] += 1
329 elif config["int_2"] == 100:
330 special_values_instances["int_2"][100] += 1
332 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_instances["int_1"][0]
333 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= (
334 special_values_instances["int_1"][1]
335 )
336 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= (
337 special_values_instances["int_2"][2]
338 )
339 assert (1 - eps) * int(num_configs * bias_percentage * 1.5) <= (
340 special_values_instances["int_2"][100]
341 )
344def test_max_unique_values_per_param() -> None:
345 """Tests LlamaTune's parameter values discretization implementation."""
346 # Define config space with a mix of different parameter types
347 input_space = CS.ConfigurationSpace(seed=1234)
348 input_space.add(
349 CS.UniformFloatHyperparameter(name="cont_1", lower=0, upper=5),
350 )
351 input_space.add(CS.UniformFloatHyperparameter(name="cont_2", lower=1, upper=100))
352 input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=1, upper=10))
353 input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=2048))
354 input_space.add(CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"]))
355 input_space.add(
356 CS.CategoricalHyperparameter(name="str_2", choices=[f"choice_{idx}" for idx in range(10)])
357 )
359 # Restrict the number of unique parameter values
360 num_configs = 200
361 for max_unique_values_per_param in (5, 25, 100):
362 adapter = LlamaTuneAdapter(
363 orig_parameter_space=input_space,
364 num_low_dims=3,
365 special_param_values=None,
366 max_unique_values_per_param=max_unique_values_per_param,
367 )
369 # Keep track of unique values generated for each parameter
370 unique_values_dict: Dict[str, set] = {param: set() for param in list(input_space.keys())}
371 for config in gen_random_configs(adapter, num_configs):
372 for param, value in config.items():
373 unique_values_dict[param].add(value)
375 # Ensure that their number is less than the maximum number allowed
376 for _, unique_values in unique_values_dict.items():
377 assert len(unique_values) <= max_unique_values_per_param
380@pytest.mark.parametrize(
381 ("num_target_space_dims", "param_space_kwargs"),
382 (
383 [
384 (num_target_space_dims, param_space_kwargs)
385 for num_target_space_dims in (2, 4)
386 for num_orig_space_factor in (1.5, 4)
387 for param_space_kwargs in (
388 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
389 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
390 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
391 {"n_quantized_integer_params": int(num_target_space_dims * num_orig_space_factor)},
392 {
393 "n_quantized_continuous_params": int(
394 num_target_space_dims * num_orig_space_factor
395 )
396 },
397 # Mix of all three types
398 {
399 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
400 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
401 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
402 },
403 )
404 ]
405 ),
406)
407def test_approx_inverse_mapping(
408 num_target_space_dims: int,
409 param_space_kwargs: dict,
410) -> None: # pylint: disable=too-many-locals
411 """Tests LlamaTune's approximate high-to-low space projection method, using pseudo-
412 inverse.
413 """
414 input_space = construct_parameter_space(**param_space_kwargs)
416 # Enable low-dimensional space projection, but disable reverse mapping
417 adapter = LlamaTuneAdapter(
418 orig_parameter_space=input_space,
419 num_low_dims=num_target_space_dims,
420 special_param_values=None,
421 max_unique_values_per_param=None,
422 use_approximate_reverse_mapping=False,
423 )
425 sampled_config = input_space.sample_configuration() # size=1)
426 with pytest.raises(ValueError):
427 sampled_config_sr = pd.Series(dict(sampled_config))
428 _ = adapter.inverse_transform(sampled_config_sr)
430 # Enable low-dimensional space projection *and* reverse mapping
431 adapter = LlamaTuneAdapter(
432 orig_parameter_space=input_space,
433 num_low_dims=num_target_space_dims,
434 special_param_values=None,
435 max_unique_values_per_param=None,
436 use_approximate_reverse_mapping=True,
437 )
439 # Warning should be printed the first time
440 sampled_config = input_space.sample_configuration() # size=1)
441 with pytest.warns(UserWarning):
442 sampled_config_sr = pd.Series(dict(sampled_config))
443 target_config_sr = adapter.inverse_transform(sampled_config_sr)
444 # Low-dim (i.e., target) config should be valid
445 target_config = CS.Configuration(
446 adapter.target_parameter_space,
447 values=target_config_sr.to_dict(),
448 )
449 target_config.check_valid_configuration()
451 # Test inverse transform with 100 random configs
452 for _ in range(100):
453 sampled_config = input_space.sample_configuration() # size=1)
454 sampled_config_sr = pd.Series(dict(sampled_config))
455 target_config_sr = adapter.inverse_transform(sampled_config_sr)
456 # Low-dim (i.e., target) config should be valid
457 target_config = CS.Configuration(
458 adapter.target_parameter_space,
459 values=target_config_sr.to_dict(),
460 )
461 target_config.check_valid_configuration()
464@pytest.mark.parametrize(
465 ("num_low_dims", "special_param_values", "max_unique_values_per_param"),
466 (
467 [
468 (num_low_dims, special_param_values, max_unique_values_per_param)
469 for num_low_dims in (8, 16)
470 for special_param_values in (
471 {"int_1": -1, "int_2": -1, "int_3": -1, "int_4": [-1, 0]},
472 {
473 "int_1": (-1, 0.1),
474 "int_2": -1,
475 "int_3": (-1, 0.3),
476 "int_4": [(-1, 0.1), (0, 0.2)],
477 },
478 )
479 for max_unique_values_per_param in (50, 250)
480 ]
481 ),
482)
483def test_llamatune_pipeline(
484 num_low_dims: int,
485 special_param_values: dict,
486 max_unique_values_per_param: int,
487) -> None:
488 """Tests LlamaTune space adapter when all components are active."""
489 # pylint: disable=too-many-locals
491 # Define config space with a mix of different parameter types
492 input_space = construct_parameter_space(
493 n_continuous_params=10,
494 n_integer_params=10,
495 n_categorical_params=5,
496 )
497 adapter = LlamaTuneAdapter(
498 orig_parameter_space=input_space,
499 num_low_dims=num_low_dims,
500 special_param_values=special_param_values,
501 max_unique_values_per_param=max_unique_values_per_param,
502 )
504 special_value_occurrences = {
505 # pylint: disable=protected-access
506 param: {special_value: 0 for special_value, _ in tuples_list}
507 for param, tuples_list in adapter._special_param_values_dict.items()
508 }
509 unique_values_dict: Dict[str, Set] = {param: set() for param in input_space.keys()}
511 num_configs = 1000
512 for (
513 config
514 ) in adapter.target_parameter_space.sample_configuration( # pylint: disable=not-an-iterable
515 size=num_configs
516 ):
517 # Transform low-dim config to high-dim point/config
518 sampled_config_sr = pd.Series(dict(config))
519 orig_config_sr = adapter.transform(sampled_config_sr)
520 # High-dim (i.e., original) config should be valid
521 orig_config = CS.Configuration(input_space, values=orig_config_sr.to_dict())
522 orig_config.check_valid_configuration()
524 # Transform high-dim config back to low-dim
525 target_config_sr = adapter.inverse_transform(orig_config_sr)
526 # Sampled config and this should be the same
527 target_config = CS.Configuration(
528 adapter.target_parameter_space,
529 values=target_config_sr.to_dict(),
530 )
531 assert target_config == config
533 for param, value in orig_config.items():
534 # Keep track of special value occurrences
535 if param in special_value_occurrences:
536 if value in special_value_occurrences[param]:
537 special_value_occurrences[param][value] += 1
539 # Keep track of unique values generated for each parameter
540 unique_values_dict[param].add(value)
542 # Ensure that occurrences of special values do not significantly deviate from expected
543 eps = 0.2
544 for (
545 param,
546 tuples_list,
547 ) in adapter._special_param_values_dict.items(): # pylint: disable=protected-access
548 for value, bias_percentage in tuples_list:
549 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences[
550 param
551 ][value]
553 # Ensure that number of unique values is less than the maximum number allowed
554 for _, unique_values in unique_values_dict.items():
555 assert len(unique_values) <= max_unique_values_per_param
558@pytest.mark.parametrize(
559 ("num_target_space_dims", "param_space_kwargs"),
560 (
561 [
562 (num_target_space_dims, param_space_kwargs)
563 for num_target_space_dims in (2, 4)
564 for num_orig_space_factor in (1.5, 4)
565 for param_space_kwargs in (
566 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
567 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
568 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
569 # Mix of all three types
570 {
571 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
572 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
573 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
574 },
575 )
576 ]
577 ),
578)
579def test_deterministic_behavior_for_same_seed(
580 num_target_space_dims: int,
581 param_space_kwargs: dict,
582) -> None:
583 """Tests LlamaTune's space adapter deterministic behavior when given same seed in
584 the input parameter space.
585 """
587 def generate_target_param_space_configs(seed: int) -> List[CS.Configuration]:
588 input_space = construct_parameter_space(**param_space_kwargs, seed=seed)
590 # Init adapter and sample points in the low-dim space
591 adapter = LlamaTuneAdapter(
592 orig_parameter_space=input_space,
593 num_low_dims=num_target_space_dims,
594 special_param_values=None,
595 max_unique_values_per_param=None,
596 use_approximate_reverse_mapping=False,
597 )
599 sample_configs: List[CS.Configuration] = (
600 adapter.target_parameter_space.sample_configuration(size=100)
601 )
602 return sample_configs
604 assert generate_target_param_space_configs(42) == generate_target_param_space_configs(42)
605 assert generate_target_param_space_configs(1234) != generate_target_param_space_configs(42)