Coverage for mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py: 100%
169 statements
« prev ^ index » next coverage.py v7.8.0, created at 2025-04-01 00:52 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Unit tests for checking tunable definition rules."""
7import json5 as json
8import pytest
10from mlos_bench.tunables.tunable import Tunable
11from mlos_bench.tunables.tunable_types import TunableValueTypeName
14def test_tunable_name() -> None:
15 """Check that tunable name is valid."""
16 with pytest.raises(ValueError):
17 # ! characters are currently disallowed in tunable names
18 Tunable(name="test!tunable", config={"type": "float", "range": [0, 1], "default": 0})
21def test_categorical_required_params() -> None:
22 """Check that required parameters are present for categorical tunables."""
23 json_config = """
24 {
25 "type": "categorical",
26 "values_missing": ["foo", "bar", "baz"],
27 "default": "foo"
28 }
29 """
30 config = json.loads(json_config)
31 assert isinstance(config, dict)
32 with pytest.raises(ValueError):
33 Tunable(name="test", config=config)
36def test_categorical_weights() -> None:
37 """Instantiate a categorical tunable with weights."""
38 json_config = """
39 {
40 "type": "categorical",
41 "values": ["foo", "bar", "baz"],
42 "values_weights": [25, 25, 50],
43 "default": "foo"
44 }
45 """
46 config = json.loads(json_config)
47 assert isinstance(config, dict)
48 tunable = Tunable(name="test", config=config)
49 assert tunable.weights == [25, 25, 50]
52def test_categorical_weights_wrong_count() -> None:
53 """Try to instantiate a categorical tunable with incorrect number of weights."""
54 json_config = """
55 {
56 "type": "categorical",
57 "values": ["foo", "bar", "baz"],
58 "values_weights": [50, 50],
59 "default": "foo"
60 }
61 """
62 config = json.loads(json_config)
63 assert isinstance(config, dict)
64 with pytest.raises(ValueError):
65 Tunable(name="test", config=config)
68def test_categorical_weights_wrong_values() -> None:
69 """Try to instantiate a categorical tunable with invalid weights."""
70 json_config = """
71 {
72 "type": "categorical",
73 "values": ["foo", "bar", "baz"],
74 "values_weights": [-1, 50, 50],
75 "default": "foo"
76 }
77 """
78 config = json.loads(json_config)
79 assert isinstance(config, dict)
80 with pytest.raises(ValueError):
81 Tunable(name="test", config=config)
84def test_categorical_wrong_params() -> None:
85 """Disallow range param for categorical tunables."""
86 json_config = """
87 {
88 "type": "categorical",
89 "values": ["foo", "bar", "foo"],
90 "range": [0, 1],
91 "default": "foo"
92 }
93 """
94 config = json.loads(json_config)
95 assert isinstance(config, dict)
96 with pytest.raises(ValueError):
97 Tunable(name="test", config=config)
100def test_categorical_disallow_special_values() -> None:
101 """Disallow special values for categorical values."""
102 json_config = """
103 {
104 "type": "categorical",
105 "values": ["foo", "bar", "foo"],
106 "special": ["baz"],
107 "default": "foo"
108 }
109 """
110 config = json.loads(json_config)
111 assert isinstance(config, dict)
112 with pytest.raises(ValueError):
113 Tunable(name="test", config=config)
116def test_categorical_tunable_disallow_repeats() -> None:
117 """Disallow duplicate values in categorical tunables."""
118 with pytest.raises(ValueError):
119 Tunable(
120 name="test",
121 config={
122 "type": "categorical",
123 "values": ["foo", "bar", "foo"],
124 "default": "foo",
125 },
126 )
129@pytest.mark.parametrize("tunable_type", ["int", "float"])
130def test_numerical_tunable_disallow_null_default(tunable_type: TunableValueTypeName) -> None:
131 """Disallow null values as default for numerical tunables."""
132 with pytest.raises(ValueError):
133 Tunable(
134 name=f"test_{tunable_type}",
135 config={
136 "type": tunable_type,
137 "range": [0, 10],
138 "default": None,
139 },
140 )
143@pytest.mark.parametrize("tunable_type", ["int", "float"])
144def test_numerical_tunable_disallow_out_of_range(tunable_type: TunableValueTypeName) -> None:
145 """Disallow out of range values as default for numerical tunables."""
146 with pytest.raises(ValueError):
147 Tunable(
148 name=f"test_{tunable_type}",
149 config={
150 "type": tunable_type,
151 "range": [0, 10],
152 "default": 11,
153 },
154 )
157@pytest.mark.parametrize("tunable_type", ["int", "float"])
158def test_numerical_tunable_wrong_params(tunable_type: TunableValueTypeName) -> None:
159 """Disallow values param for numerical tunables."""
160 with pytest.raises(ValueError):
161 Tunable(
162 name=f"test_{tunable_type}",
163 config={
164 "type": tunable_type,
165 "range": [0, 10],
166 "values": ["foo", "bar"],
167 "default": 0,
168 },
169 )
172@pytest.mark.parametrize("tunable_type", ["int", "float"])
173def test_numerical_tunable_required_params(tunable_type: TunableValueTypeName) -> None:
174 """Disallow null values param for numerical tunables."""
175 json_config = f"""
176 {
177 "type": "{tunable_type}",
178 "range_missing": [0, 10],
179 "default": 0
180 }
181 """
182 config = json.loads(json_config)
183 assert isinstance(config, dict)
184 with pytest.raises(ValueError):
185 Tunable(name=f"test_{tunable_type}", config=config)
188@pytest.mark.parametrize("tunable_type", ["int", "float"])
189def test_numerical_tunable_invalid_range(tunable_type: TunableValueTypeName) -> None:
190 """Disallow invalid range param for numerical tunables."""
191 json_config = f"""
192 {
193 "type": "{tunable_type}",
194 "range": [0, 10, 7],
195 "default": 0
196 }
197 """
198 config = json.loads(json_config)
199 assert isinstance(config, dict)
200 with pytest.raises(AssertionError):
201 Tunable(name=f"test_{tunable_type}", config=config)
204@pytest.mark.parametrize("tunable_type", ["int", "float"])
205def test_numerical_tunable_reversed_range(tunable_type: TunableValueTypeName) -> None:
206 """Disallow reverse range param for numerical tunables."""
207 json_config = f"""
208 {
209 "type": "{tunable_type}",
210 "range": [10, 0],
211 "default": 0
212 }
213 """
214 config = json.loads(json_config)
215 assert isinstance(config, dict)
216 with pytest.raises(ValueError):
217 Tunable(name=f"test_{tunable_type}", config=config)
220@pytest.mark.parametrize("tunable_type", ["int", "float"])
221def test_numerical_weights(tunable_type: TunableValueTypeName) -> None:
222 """Instantiate a numerical tunable with weighted special values."""
223 json_config = f"""
224 {
225 "type": "{tunable_type}",
226 "range": [0, 100],
227 "special": [0],
228 "special_weights": [0.1],
229 "range_weight": 0.9,
230 "default": 0
231 }
232 """
233 config = json.loads(json_config)
234 assert isinstance(config, dict)
235 tunable = Tunable(name="test", config=config)
236 assert tunable.special == [0]
237 assert tunable.weights == [0.1]
238 assert tunable.range_weight == 0.9
241@pytest.mark.parametrize("tunable_type", ["int", "float"])
242def test_numerical_quantization(tunable_type: TunableValueTypeName) -> None:
243 """Instantiate a numerical tunable with quantization."""
244 json_config = f"""
245 {
246 "type": "{tunable_type}",
247 "range": [0, 100],
248 "quantization_bins": 11,
249 "default": 0
250 }
251 """
252 config = json.loads(json_config)
253 assert isinstance(config, dict)
254 tunable = Tunable(name="test", config=config)
255 expected = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
256 assert tunable.quantization_bins == len(expected)
257 assert pytest.approx(list(tunable.quantized_values or []), 1e-8) == expected
258 assert not tunable.is_log
261@pytest.mark.parametrize("tunable_type", ["int", "float"])
262def test_numerical_log(tunable_type: TunableValueTypeName) -> None:
263 """Instantiate a numerical tunable with log scale."""
264 json_config = f"""
265 {
266 "type": "{tunable_type}",
267 "range": [0, 100],
268 "log": true,
269 "default": 0
270 }
271 """
272 config = json.loads(json_config)
273 assert isinstance(config, dict)
274 tunable = Tunable(name="test", config=config)
275 assert tunable.is_log
278@pytest.mark.parametrize("tunable_type", ["int", "float"])
279def test_numerical_weights_no_specials(tunable_type: TunableValueTypeName) -> None:
280 """Raise an error if special_weights are specified but no special values."""
281 json_config = f"""
282 {
283 "type": "{tunable_type}",
284 "range": [0, 100],
285 "special_weights": [0.1, 0.9],
286 "default": 0
287 }
288 """
289 config = json.loads(json_config)
290 assert isinstance(config, dict)
291 with pytest.raises(ValueError):
292 Tunable(name="test", config=config)
295@pytest.mark.parametrize("tunable_type", ["int", "float"])
296def test_numerical_weights_non_normalized(tunable_type: TunableValueTypeName) -> None:
297 """Instantiate a numerical tunable with non-normalized weights of the special
298 values.
299 """
300 json_config = f"""
301 {
302 "type": "{tunable_type}",
303 "range": [0, 100],
304 "special": [-1, 0],
305 "special_weights": [0, 10],
306 "range_weight": 90,
307 "default": 0
308 }
309 """
310 config = json.loads(json_config)
311 assert isinstance(config, dict)
312 tunable = Tunable(name="test", config=config)
313 assert tunable.special == [-1, 0]
314 assert tunable.weights == [0, 10] # Zero weights are ok
315 assert tunable.range_weight == 90
318@pytest.mark.parametrize("tunable_type", ["int", "float"])
319def test_numerical_weights_wrong_count(tunable_type: TunableValueTypeName) -> None:
320 """Try to instantiate a numerical tunable with incorrect number of weights."""
321 json_config = f"""
322 {
323 "type": "{tunable_type}",
324 "range": [0, 100],
325 "special": [0],
326 "special_weights": [0.1, 0.1, 0.8],
327 "range_weight": 0.1,
328 "default": 0
329 }
330 """
331 config = json.loads(json_config)
332 assert isinstance(config, dict)
333 with pytest.raises(ValueError):
334 Tunable(name="test", config=config)
337@pytest.mark.parametrize("tunable_type", ["int", "float"])
338def test_numerical_weights_no_range_weight(tunable_type: TunableValueTypeName) -> None:
339 """Try to instantiate a numerical tunable with weights but no range_weight."""
340 json_config = f"""
341 {
342 "type": "{tunable_type}",
343 "range": [0, 100],
344 "special": [0, -1],
345 "special_weights": [0.1, 0.2],
346 "default": 0
347 }
348 """
349 config = json.loads(json_config)
350 assert isinstance(config, dict)
351 with pytest.raises(ValueError):
352 Tunable(name="test", config=config)
355@pytest.mark.parametrize("tunable_type", ["int", "float"])
356def test_numerical_range_weight_no_weights(tunable_type: TunableValueTypeName) -> None:
357 """Try to instantiate a numerical tunable with specials but no range_weight."""
358 json_config = f"""
359 {
360 "type": "{tunable_type}",
361 "range": [0, 100],
362 "special": [0, -1],
363 "range_weight": 0.3,
364 "default": 0
365 }
366 """
367 config = json.loads(json_config)
368 assert isinstance(config, dict)
369 with pytest.raises(ValueError):
370 Tunable(name="test", config=config)
373@pytest.mark.parametrize("tunable_type", ["int", "float"])
374def test_numerical_range_weight_no_specials(tunable_type: TunableValueTypeName) -> None:
375 """Try to instantiate a numerical tunable with specials but no range_weight."""
376 json_config = f"""
377 {
378 "type": "{tunable_type}",
379 "range": [0, 100],
380 "range_weight": 0.3,
381 "default": 0
382 }
383 """
384 config = json.loads(json_config)
385 assert isinstance(config, dict)
386 with pytest.raises(ValueError):
387 Tunable(name="test", config=config)
390@pytest.mark.parametrize("tunable_type", ["int", "float"])
391def test_numerical_weights_wrong_values(tunable_type: TunableValueTypeName) -> None:
392 """Try to instantiate a numerical tunable with incorrect number of weights."""
393 json_config = f"""
394 {
395 "type": "{tunable_type}",
396 "range": [0, 100],
397 "special": [0],
398 "special_weights": [-1],
399 "range_weight": 10,
400 "default": 0
401 }
402 """
403 config = json.loads(json_config)
404 assert isinstance(config, dict)
405 with pytest.raises(ValueError):
406 Tunable(name="test", config=config)
409@pytest.mark.parametrize("tunable_type", ["int", "float"])
410def test_numerical_quantization_wrong(tunable_type: TunableValueTypeName) -> None:
411 """Instantiate a numerical tunable with invalid number of quantization points."""
412 json_config = f"""
413 {
414 "type": "{tunable_type}",
415 "range": [0, 100],
416 "quantization_bins": 0,
417 "default": 0
418 }
419 """
420 config = json.loads(json_config)
421 assert isinstance(config, dict)
422 with pytest.raises(ValueError):
423 Tunable(name="test", config=config)
426def test_bad_type() -> None:
427 """Disallow bad types."""
428 json_config = """
429 {
430 "type": "foo",
431 "range": [0, 10],
432 "default": 0
433 }
434 """
435 config = json.loads(json_config)
436 assert isinstance(config, dict)
437 with pytest.raises(ValueError):
438 Tunable(name="test_bad_type", config=config)