Coverage for mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py: 100%
146 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-05 00:36 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""
6Unit tests for checking tunable definition rules.
7"""
9import json5 as json
10import pytest
12from mlos_bench.tunables.tunable import Tunable, TunableValueTypeName
15def test_tunable_name() -> None:
16 """
17 Check that tunable name is valid.
18 """
19 with pytest.raises(ValueError):
20 # ! characters are currently disallowed in tunable names
21 Tunable(name='test!tunable', config={"type": "float", "range": [0, 1], "default": 0})
24def test_categorical_required_params() -> None:
25 """
26 Check that required parameters are present for categorical tunables.
27 """
28 json_config = """
29 {
30 "type": "categorical",
31 "values_missing": ["foo", "bar", "baz"],
32 "default": "foo"
33 }
34 """
35 config = json.loads(json_config)
36 with pytest.raises(ValueError):
37 Tunable(name='test', config=config)
40def test_categorical_weights() -> None:
41 """
42 Instantiate a categorical tunable with weights.
43 """
44 json_config = """
45 {
46 "type": "categorical",
47 "values": ["foo", "bar", "baz"],
48 "values_weights": [25, 25, 50],
49 "default": "foo"
50 }
51 """
52 config = json.loads(json_config)
53 tunable = Tunable(name='test', config=config)
54 assert tunable.weights == [25, 25, 50]
57def test_categorical_weights_wrong_count() -> None:
58 """
59 Try to instantiate a categorical tunable with incorrect number of weights.
60 """
61 json_config = """
62 {
63 "type": "categorical",
64 "values": ["foo", "bar", "baz"],
65 "values_weights": [50, 50],
66 "default": "foo"
67 }
68 """
69 config = json.loads(json_config)
70 with pytest.raises(ValueError):
71 Tunable(name='test', config=config)
74def test_categorical_weights_wrong_values() -> None:
75 """
76 Try to instantiate a categorical tunable with invalid weights.
77 """
78 json_config = """
79 {
80 "type": "categorical",
81 "values": ["foo", "bar", "baz"],
82 "values_weights": [-1, 50, 50],
83 "default": "foo"
84 }
85 """
86 config = json.loads(json_config)
87 with pytest.raises(ValueError):
88 Tunable(name='test', config=config)
91def test_categorical_wrong_params() -> None:
92 """
93 Disallow range param for categorical tunables.
94 """
95 json_config = """
96 {
97 "type": "categorical",
98 "values": ["foo", "bar", "foo"],
99 "range": [0, 1],
100 "default": "foo"
101 }
102 """
103 config = json.loads(json_config)
104 with pytest.raises(ValueError):
105 Tunable(name='test', config=config)
108def test_categorical_disallow_special_values() -> None:
109 """
110 Disallow special values for categorical values.
111 """
112 json_config = """
113 {
114 "type": "categorical",
115 "values": ["foo", "bar", "foo"],
116 "special": ["baz"],
117 "default": "foo"
118 }
119 """
120 config = json.loads(json_config)
121 with pytest.raises(ValueError):
122 Tunable(name='test', config=config)
125def test_categorical_tunable_disallow_repeats() -> None:
126 """
127 Disallow duplicate values in categorical tunables.
128 """
129 with pytest.raises(ValueError):
130 Tunable(name='test', config={
131 "type": "categorical",
132 "values": ["foo", "bar", "foo"],
133 "default": "foo",
134 })
137@pytest.mark.parametrize("tunable_type", ["int", "float"])
138def test_numerical_tunable_disallow_null_default(tunable_type: TunableValueTypeName) -> None:
139 """
140 Disallow null values as default for numerical tunables.
141 """
142 with pytest.raises(ValueError):
143 Tunable(name=f'test_{tunable_type}', config={
144 "type": tunable_type,
145 "range": [0, 10],
146 "default": None,
147 })
150@pytest.mark.parametrize("tunable_type", ["int", "float"])
151def test_numerical_tunable_disallow_out_of_range(tunable_type: TunableValueTypeName) -> None:
152 """
153 Disallow out of range values as default for numerical tunables.
154 """
155 with pytest.raises(ValueError):
156 Tunable(name=f'test_{tunable_type}', config={
157 "type": tunable_type,
158 "range": [0, 10],
159 "default": 11,
160 })
163@pytest.mark.parametrize("tunable_type", ["int", "float"])
164def test_numerical_tunable_wrong_params(tunable_type: TunableValueTypeName) -> None:
165 """
166 Disallow values param for numerical tunables.
167 """
168 with pytest.raises(ValueError):
169 Tunable(name=f'test_{tunable_type}', config={
170 "type": tunable_type,
171 "range": [0, 10],
172 "values": ["foo", "bar"],
173 "default": 0,
174 })
177@pytest.mark.parametrize("tunable_type", ["int", "float"])
178def test_numerical_tunable_required_params(tunable_type: TunableValueTypeName) -> None:
179 """
180 Disallow null values param for numerical tunables.
181 """
182 json_config = f"""
183 {{
184 "type": "{tunable_type}",
185 "range_missing": [0, 10],
186 "default": 0
187 }}
188 """
189 config = json.loads(json_config)
190 with pytest.raises(ValueError):
191 Tunable(name=f'test_{tunable_type}', config=config)
194@pytest.mark.parametrize("tunable_type", ["int", "float"])
195def test_numerical_tunable_invalid_range(tunable_type: TunableValueTypeName) -> None:
196 """
197 Disallow invalid range param for numerical tunables.
198 """
199 json_config = f"""
200 {{
201 "type": "{tunable_type}",
202 "range": [0, 10, 7],
203 "default": 0
204 }}
205 """
206 config = json.loads(json_config)
207 with pytest.raises(AssertionError):
208 Tunable(name=f'test_{tunable_type}', config=config)
211@pytest.mark.parametrize("tunable_type", ["int", "float"])
212def test_numerical_tunable_reversed_range(tunable_type: TunableValueTypeName) -> None:
213 """
214 Disallow reverse range param for numerical tunables.
215 """
216 json_config = f"""
217 {{
218 "type": "{tunable_type}",
219 "range": [10, 0],
220 "default": 0
221 }}
222 """
223 config = json.loads(json_config)
224 with pytest.raises(ValueError):
225 Tunable(name=f'test_{tunable_type}', config=config)
228@pytest.mark.parametrize("tunable_type", ["int", "float"])
229def test_numerical_weights(tunable_type: TunableValueTypeName) -> None:
230 """
231 Instantiate a numerical tunable with weighted special values.
232 """
233 json_config = f"""
234 {{
235 "type": "{tunable_type}",
236 "range": [0, 100],
237 "special": [0],
238 "special_weights": [0.1],
239 "range_weight": 0.9,
240 "default": 0
241 }}
242 """
243 config = json.loads(json_config)
244 tunable = Tunable(name='test', config=config)
245 assert tunable.special == [0]
246 assert tunable.weights == [0.1]
247 assert tunable.range_weight == 0.9
250@pytest.mark.parametrize("tunable_type", ["int", "float"])
251def test_numerical_quantization(tunable_type: TunableValueTypeName) -> None:
252 """
253 Instantiate a numerical tunable with quantization.
254 """
255 json_config = f"""
256 {{
257 "type": "{tunable_type}",
258 "range": [0, 100],
259 "quantization": 10,
260 "default": 0
261 }}
262 """
263 config = json.loads(json_config)
264 tunable = Tunable(name='test', config=config)
265 assert tunable.quantization == 10
266 assert not tunable.is_log
269@pytest.mark.parametrize("tunable_type", ["int", "float"])
270def test_numerical_log(tunable_type: TunableValueTypeName) -> None:
271 """
272 Instantiate a numerical tunable with log scale.
273 """
274 json_config = f"""
275 {{
276 "type": "{tunable_type}",
277 "range": [0, 100],
278 "log": true,
279 "default": 0
280 }}
281 """
282 config = json.loads(json_config)
283 tunable = Tunable(name='test', config=config)
284 assert tunable.is_log
287@pytest.mark.parametrize("tunable_type", ["int", "float"])
288def test_numerical_weights_no_specials(tunable_type: TunableValueTypeName) -> None:
289 """
290 Raise an error if special_weights are specified but no special values.
291 """
292 json_config = f"""
293 {{
294 "type": "{tunable_type}",
295 "range": [0, 100],
296 "special_weights": [0.1, 0.9],
297 "default": 0
298 }}
299 """
300 config = json.loads(json_config)
301 with pytest.raises(ValueError):
302 Tunable(name='test', config=config)
305@pytest.mark.parametrize("tunable_type", ["int", "float"])
306def test_numerical_weights_non_normalized(tunable_type: TunableValueTypeName) -> None:
307 """
308 Instantiate a numerical tunable with non-normalized weights
309 of the special values.
310 """
311 json_config = f"""
312 {{
313 "type": "{tunable_type}",
314 "range": [0, 100],
315 "special": [-1, 0],
316 "special_weights": [0, 10],
317 "range_weight": 90,
318 "default": 0
319 }}
320 """
321 config = json.loads(json_config)
322 tunable = Tunable(name='test', config=config)
323 assert tunable.special == [-1, 0]
324 assert tunable.weights == [0, 10] # Zero weights are ok
325 assert tunable.range_weight == 90
328@pytest.mark.parametrize("tunable_type", ["int", "float"])
329def test_numerical_weights_wrong_count(tunable_type: TunableValueTypeName) -> None:
330 """
331 Try to instantiate a numerical tunable with incorrect number of weights.
332 """
333 json_config = f"""
334 {{
335 "type": "{tunable_type}",
336 "range": [0, 100],
337 "special": [0],
338 "special_weights": [0.1, 0.1, 0.8],
339 "range_weight": 0.1,
340 "default": 0
341 }}
342 """
343 config = json.loads(json_config)
344 with pytest.raises(ValueError):
345 Tunable(name='test', config=config)
348@pytest.mark.parametrize("tunable_type", ["int", "float"])
349def test_numerical_weights_no_range_weight(tunable_type: TunableValueTypeName) -> None:
350 """
351 Try to instantiate a numerical tunable with weights but no range_weight.
352 """
353 json_config = f"""
354 {{
355 "type": "{tunable_type}",
356 "range": [0, 100],
357 "special": [0, -1],
358 "special_weights": [0.1, 0.2],
359 "default": 0
360 }}
361 """
362 config = json.loads(json_config)
363 with pytest.raises(ValueError):
364 Tunable(name='test', config=config)
367@pytest.mark.parametrize("tunable_type", ["int", "float"])
368def test_numerical_range_weight_no_weights(tunable_type: TunableValueTypeName) -> None:
369 """
370 Try to instantiate a numerical tunable with specials but no range_weight.
371 """
372 json_config = f"""
373 {{
374 "type": "{tunable_type}",
375 "range": [0, 100],
376 "special": [0, -1],
377 "range_weight": 0.3,
378 "default": 0
379 }}
380 """
381 config = json.loads(json_config)
382 with pytest.raises(ValueError):
383 Tunable(name='test', config=config)
386@pytest.mark.parametrize("tunable_type", ["int", "float"])
387def test_numerical_range_weight_no_specials(tunable_type: TunableValueTypeName) -> None:
388 """
389 Try to instantiate a numerical tunable with specials but no range_weight.
390 """
391 json_config = f"""
392 {{
393 "type": "{tunable_type}",
394 "range": [0, 100],
395 "range_weight": 0.3,
396 "default": 0
397 }}
398 """
399 config = json.loads(json_config)
400 with pytest.raises(ValueError):
401 Tunable(name='test', config=config)
404@pytest.mark.parametrize("tunable_type", ["int", "float"])
405def test_numerical_weights_wrong_values(tunable_type: TunableValueTypeName) -> None:
406 """
407 Try to instantiate a numerical tunable with incorrect number of weights.
408 """
409 json_config = f"""
410 {{
411 "type": "{tunable_type}",
412 "range": [0, 100],
413 "special": [0],
414 "special_weights": [-1],
415 "range_weight": 10,
416 "default": 0
417 }}
418 """
419 config = json.loads(json_config)
420 with pytest.raises(ValueError):
421 Tunable(name='test', config=config)
424@pytest.mark.parametrize("tunable_type", ["int", "float"])
425def test_numerical_quantization_wrong(tunable_type: TunableValueTypeName) -> None:
426 """
427 Instantiate a numerical tunable with invalid number of quantization points.
428 """
429 json_config = f"""
430 {{
431 "type": "{tunable_type}",
432 "range": [0, 100],
433 "quantization": 0,
434 "default": 0
435 }}
436 """
437 config = json.loads(json_config)
438 with pytest.raises(ValueError):
439 Tunable(name='test', config=config)
442def test_bad_type() -> None:
443 """
444 Disallow bad types.
445 """
446 json_config = """
447 {
448 "type": "foo",
449 "range": [0, 10],
450 "default": 0
451 }
452 """
453 config = json.loads(json_config)
454 with pytest.raises(ValueError):
455 Tunable(name='test_bad_type', config=config)