Coverage for mlos_core/mlos_core/tests/spaces/spaces_test.py: 94%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Tests for mlos_core.spaces 

7""" 

8 

9# pylint: disable=missing-function-docstring 

10 

11from abc import ABCMeta, abstractmethod 

12from typing import Any, Callable, List, NoReturn, Union 

13 

14import numpy as np 

15import numpy.typing as npt 

16import pytest 

17 

18import scipy 

19 

20import ConfigSpace as CS 

21from ConfigSpace.hyperparameters import NormalIntegerHyperparameter 

22 

23import flaml.tune.sample 

24 

25from mlos_core.spaces.converters.flaml import configspace_to_flaml_space, FlamlDomain, FlamlSpace 

26 

27 

28OptimizerSpace = Union[FlamlSpace, CS.ConfigurationSpace] 

29OptimizerParam = Union[FlamlDomain, CS.hyperparameters.Hyperparameter] 

30 

31 

32def assert_is_uniform(arr: npt.NDArray) -> None: 

33 """Implements a few tests for uniformity.""" 

34 _values, counts = np.unique(arr, return_counts=True) 

35 

36 kurtosis = scipy.stats.kurtosis(arr) 

37 

38 _chi_sq, p_value = scipy.stats.chisquare(counts) 

39 

40 frequencies = counts / len(arr) 

41 assert np.isclose(frequencies.sum(), 1) 

42 _f_chi_sq, f_p_value = scipy.stats.chisquare(frequencies) 

43 

44 assert np.isclose(kurtosis, -1.2, atol=.1) 

45 assert p_value > .3 

46 assert f_p_value > .5 

47 

48 

49def assert_is_log_uniform(arr: npt.NDArray, base: float = np.e) -> None: 

50 """Checks whether an array is log uniformly distributed.""" 

51 logs = np.log(arr) / np.log(base) 

52 assert_is_uniform(logs) 

53 

54 

55def test_is_uniform() -> None: 

56 """Test our uniform distribution check function.""" 

57 np.random.seed(42) 

58 uniform = np.random.uniform(1, 20, 1000) 

59 assert_is_uniform(uniform) 

60 

61 

62def test_is_log_uniform() -> None: 

63 """Test our log uniform distribution check function.""" 

64 np.random.seed(42) 

65 log_uniform = np.exp(np.random.uniform(np.log(1), np.log(20), 1000)) 

66 assert_is_log_uniform(log_uniform) 

67 

68 

69def invalid_conversion_function(*args: Any) -> NoReturn: 

70 """ 

71 A quick dummy function for the base class to make pylint happy. 

72 """ 

73 raise NotImplementedError('subclass must override conversion_function') 

74 

75 

76class BaseConversion(metaclass=ABCMeta): 

77 """ 

78 Base class for testing optimizer space conversions. 

79 """ 

80 conversion_function: Callable[..., OptimizerSpace] = invalid_conversion_function 

81 

82 @abstractmethod 

83 def sample(self, config_space: OptimizerSpace, n_samples: int = 1) -> OptimizerParam: 

84 """ 

85 Sample from the given configuration space. 

86 

87 Parameters 

88 ---------- 

89 config_space : CS.ConfigurationSpace 

90 Configuration space to sample from. 

91 n_samples : int, optional 

92 Number of samples to use, by default 1. 

93 """ 

94 

95 @abstractmethod 

96 def get_parameter_names(self, config_space: OptimizerSpace) -> List[str]: 

97 """ 

98 Get the parameter names from the given configuration space. 

99 

100 Parameters 

101 ---------- 

102 config_space : CS.ConfigurationSpace 

103 Configuration space. 

104 """ 

105 

106 @abstractmethod 

107 def categorical_counts(self, points: npt.NDArray) -> npt.NDArray: 

108 """ 

109 Get the counts of each categorical value in the given points. 

110 

111 Parameters 

112 ---------- 

113 points : np.array 

114 Counts of each categorical value. 

115 """ 

116 

117 @abstractmethod 

118 def test_dimensionality(self) -> None: 

119 """ 

120 Check that the dimensionality of the converted space is correct. 

121 """ 

122 

123 def test_unsupported_hyperparameter(self) -> None: 

124 input_space = CS.ConfigurationSpace() 

125 input_space.add_hyperparameter(NormalIntegerHyperparameter("a", 2, 1)) 

126 with pytest.raises(ValueError, match="NormalIntegerHyperparameter"): 

127 self.conversion_function(input_space) 

128 

129 def test_continuous_bounds(self) -> None: 

130 input_space = CS.ConfigurationSpace() 

131 input_space.add_hyperparameter(CS.UniformFloatHyperparameter("a", lower=100, upper=200)) 

132 input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("b", lower=-10, upper=-5)) 

133 

134 converted_space = self.conversion_function(input_space) 

135 assert self.get_parameter_names(converted_space) == ["a", "b"] 

136 point = self.sample(converted_space) 

137 assert 100 <= point[0] <= 200 

138 assert -10 <= point[1] <= -5 

139 

140 def test_uniform_samples(self) -> None: 

141 input_space = CS.ConfigurationSpace() 

142 input_space.add_hyperparameter(CS.UniformFloatHyperparameter("a", lower=1, upper=5)) 

143 input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("c", lower=1, upper=20)) 

144 converted_space = self.conversion_function(input_space) 

145 

146 np.random.seed(42) 

147 uniform, integer_uniform = self.sample(converted_space, n_samples=1000).T 

148 

149 # uniform float 

150 assert_is_uniform(uniform) 

151 

152 # Check that we get both ends of the sampled range returned to us. 

153 assert input_space['c'].lower in integer_uniform 

154 assert input_space['c'].upper in integer_uniform 

155 # integer uniform 

156 assert_is_uniform(integer_uniform) 

157 

158 def test_uniform_categorical(self) -> None: 

159 input_space = CS.ConfigurationSpace() 

160 input_space.add_hyperparameter(CS.CategoricalHyperparameter("c", choices=["foo", "bar"])) 

161 converted_space = self.conversion_function(input_space) 

162 points = self.sample(converted_space, n_samples=100) 

163 counts = self.categorical_counts(points) 

164 assert 35 < counts[0] < 65 

165 assert 35 < counts[1] < 65 

166 

167 def test_weighted_categorical(self) -> None: 

168 raise NotImplementedError('subclass must override') 

169 

170 def test_log_int_spaces(self) -> None: 

171 raise NotImplementedError('subclass must override') 

172 

173 def test_log_float_spaces(self) -> None: 

174 raise NotImplementedError('subclass must override') 

175 

176 

177class TestFlamlConversion(BaseConversion): 

178 """ 

179 Tests for ConfigSpace to Flaml parameter conversions. 

180 """ 

181 

182 conversion_function = staticmethod(configspace_to_flaml_space) 

183 

184 def sample(self, config_space: FlamlSpace, n_samples: int = 1) -> npt.NDArray: # type: ignore[override] 

185 assert isinstance(config_space, dict) 

186 assert isinstance(next(iter(config_space.values())), flaml.tune.sample.Domain) 

187 ret: npt.NDArray = np.array([domain.sample(size=n_samples) for domain in config_space.values()]).T 

188 return ret 

189 

190 def get_parameter_names(self, config_space: FlamlSpace) -> List[str]: # type: ignore[override] 

191 assert isinstance(config_space, dict) 

192 ret: List[str] = list(config_space.keys()) 

193 return ret 

194 

195 def categorical_counts(self, points: npt.NDArray) -> npt.NDArray: 

196 _vals, counts = np.unique(points, return_counts=True) 

197 assert isinstance(counts, np.ndarray) 

198 return counts 

199 

200 def test_dimensionality(self) -> None: 

201 input_space = CS.ConfigurationSpace() 

202 input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("a", lower=1, upper=10)) 

203 input_space.add_hyperparameter(CS.CategoricalHyperparameter("b", choices=["bof", "bum"])) 

204 input_space.add_hyperparameter(CS.CategoricalHyperparameter("c", choices=["foo", "bar"])) 

205 output_space = configspace_to_flaml_space(input_space) 

206 assert len(output_space) == 3 

207 

208 def test_weighted_categorical(self) -> None: 

209 np.random.seed(42) 

210 input_space = CS.ConfigurationSpace() 

211 input_space.add_hyperparameter(CS.CategoricalHyperparameter("c", choices=["foo", "bar"], weights=[0.9, 0.1])) 

212 with pytest.raises(ValueError, match="non-uniform"): 

213 configspace_to_flaml_space(input_space) 

214 

215 @pytest.mark.skip(reason="FIXME: flaml sampling is non-log-uniform") 

216 def test_log_int_spaces(self) -> None: 

217 np.random.seed(42) 

218 # integer is supported 

219 input_space = CS.ConfigurationSpace() 

220 input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("d", lower=1, upper=20, log=True)) 

221 converted_space = configspace_to_flaml_space(input_space) 

222 

223 # test log integer sampling 

224 integer_log_uniform = self.sample(converted_space, n_samples=1000) 

225 integer_log_uniform = np.array(integer_log_uniform).ravel() 

226 

227 # FIXME: this fails - flaml is calling np.random.uniform() on base 10 

228 # logs of the bounds as expected but for some reason the resulting 

229 # samples are more skewed towards the lower end of the range 

230 # See Also: https://github.com/microsoft/FLAML/issues/1104 

231 assert_is_log_uniform(integer_log_uniform, base=10) 

232 

233 def test_log_float_spaces(self) -> None: 

234 np.random.seed(42) 

235 

236 # continuous is supported 

237 input_space = CS.ConfigurationSpace() 

238 input_space.add_hyperparameter(CS.UniformFloatHyperparameter("b", lower=1, upper=5, log=True)) 

239 converted_space = configspace_to_flaml_space(input_space) 

240 

241 # test log integer sampling 

242 float_log_uniform = self.sample(converted_space, n_samples=1000) 

243 float_log_uniform = np.array(float_log_uniform).ravel() 

244 

245 assert_is_log_uniform(float_log_uniform) 

246 

247 

248if __name__ == '__main__': 

249 # For attaching debugger debugging: 

250 pytest.main(["-vv", "-k", "test_log_int_spaces", __file__])