Coverage for mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for Tunable to ConfigSpace conversion. 

7""" 

8 

9import pytest 

10 

11from ConfigSpace import ( 

12 CategoricalHyperparameter, 

13 ConfigurationSpace, 

14 EqualsCondition, 

15 UniformFloatHyperparameter, 

16 UniformIntegerHyperparameter, 

17) 

18 

19from mlos_bench.tunables.tunable import Tunable 

20from mlos_bench.tunables.tunable_groups import TunableGroups 

21from mlos_bench.optimizers.convert_configspace import ( 

22 TunableValueKind, 

23 _tunable_to_configspace, 

24 special_param_names, 

25 tunable_groups_to_configspace, 

26) 

27 

28# pylint: disable=redefined-outer-name 

29 

30 

31@pytest.fixture 

32def configuration_space() -> ConfigurationSpace: 

33 """ 

34 A test fixture that produces a mock ConfigurationSpace object 

35 matching the tunable_groups fixture. 

36 

37 Returns 

38 ------- 

39 configuration_space : ConfigurationSpace 

40 A new ConfigurationSpace object for testing. 

41 """ 

42 (kernel_sched_migration_cost_ns_special, 

43 kernel_sched_migration_cost_ns_type) = special_param_names("kernel_sched_migration_cost_ns") 

44 

45 spaces = ConfigurationSpace(space={ 

46 "vmSize": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"], 

47 "idle": ["halt", "mwait", "noidle"], 

48 "kernel_sched_migration_cost_ns": (0, 500000), 

49 kernel_sched_migration_cost_ns_special: [-1, 0], 

50 kernel_sched_migration_cost_ns_type: [TunableValueKind.SPECIAL, TunableValueKind.RANGE], 

51 "kernel_sched_latency_ns": (0, 1000000000), 

52 }) 

53 

54 # NOTE: FLAML requires distribution to be uniform 

55 spaces["vmSize"].default_value = "Standard_B4ms" 

56 spaces["idle"].default_value = "halt" 

57 spaces["kernel_sched_migration_cost_ns"].default_value = 250000 

58 spaces[kernel_sched_migration_cost_ns_special].default_value = -1 

59 spaces[kernel_sched_migration_cost_ns_special].probabilities = (0.5, 0.5) 

60 spaces[kernel_sched_migration_cost_ns_type].default_value = TunableValueKind.SPECIAL 

61 spaces[kernel_sched_migration_cost_ns_type].probabilities = (0.5, 0.5) 

62 spaces["kernel_sched_latency_ns"].default_value = 2000000 

63 

64 spaces.add_condition(EqualsCondition( 

65 spaces[kernel_sched_migration_cost_ns_special], 

66 spaces[kernel_sched_migration_cost_ns_type], TunableValueKind.SPECIAL)) 

67 spaces.add_condition(EqualsCondition( 

68 spaces["kernel_sched_migration_cost_ns"], 

69 spaces[kernel_sched_migration_cost_ns_type], TunableValueKind.RANGE)) 

70 

71 return spaces 

72 

73 

74def _cmp_tunable_hyperparameter_categorical( 

75 tunable: Tunable, space: ConfigurationSpace) -> None: 

76 """ 

77 Check if categorical Tunable and ConfigSpace Hyperparameter actually match. 

78 """ 

79 param = space[tunable.name] 

80 assert isinstance(param, CategoricalHyperparameter) 

81 assert set(param.choices) == set(tunable.categories) 

82 assert param.default_value == tunable.value 

83 

84 

85def _cmp_tunable_hyperparameter_numerical( 

86 tunable: Tunable, space: ConfigurationSpace) -> None: 

87 """ 

88 Check if integer Tunable and ConfigSpace Hyperparameter actually match. 

89 """ 

90 param = space[tunable.name] 

91 assert isinstance(param, (UniformIntegerHyperparameter, UniformFloatHyperparameter)) 

92 assert (param.lower, param.upper) == tuple(tunable.range) 

93 if tunable.in_range(tunable.value): 

94 assert param.default_value == tunable.value 

95 

96 

97def test_tunable_to_configspace_categorical(tunable_categorical: Tunable) -> None: 

98 """ 

99 Check the conversion of Tunable to CategoricalHyperparameter. 

100 """ 

101 cs_param = _tunable_to_configspace(tunable_categorical) 

102 _cmp_tunable_hyperparameter_categorical(tunable_categorical, cs_param) 

103 

104 

105def test_tunable_to_configspace_int(tunable_int: Tunable) -> None: 

106 """ 

107 Check the conversion of Tunable to UniformIntegerHyperparameter. 

108 """ 

109 cs_param = _tunable_to_configspace(tunable_int) 

110 _cmp_tunable_hyperparameter_numerical(tunable_int, cs_param) 

111 

112 

113def test_tunable_to_configspace_float(tunable_float: Tunable) -> None: 

114 """ 

115 Check the conversion of Tunable to UniformFloatHyperparameter. 

116 """ 

117 cs_param = _tunable_to_configspace(tunable_float) 

118 _cmp_tunable_hyperparameter_numerical(tunable_float, cs_param) 

119 

120 

121_CMP_FUNC = { 

122 "int": _cmp_tunable_hyperparameter_numerical, 

123 "float": _cmp_tunable_hyperparameter_numerical, 

124 "categorical": _cmp_tunable_hyperparameter_categorical, 

125} 

126 

127 

128def test_tunable_groups_to_hyperparameters(tunable_groups: TunableGroups) -> None: 

129 """ 

130 Check the conversion of TunableGroups to ConfigurationSpace. 

131 Make sure that the corresponding Tunable and Hyperparameter objects match. 

132 """ 

133 space = tunable_groups_to_configspace(tunable_groups) 

134 for (tunable, _group) in tunable_groups: 

135 _CMP_FUNC[tunable.type](tunable, space) 

136 

137 

138def test_tunable_groups_to_configspace( 

139 tunable_groups: TunableGroups, configuration_space: ConfigurationSpace) -> None: 

140 """ 

141 Check the conversion of the entire TunableGroups collection 

142 to a single ConfigurationSpace object. 

143 """ 

144 space = tunable_groups_to_configspace(tunable_groups) 

145 assert space == configuration_space