Coverage for mlos_bench/mlos_bench/tests/launcher_in_process_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests to check the launcher and the main optimization loop in-process.""" 

6 

7from typing import List 

8 

9import pytest 

10 

11from mlos_bench.run import _main 

12 

13 

14@pytest.mark.parametrize( 

15 ("argv", "expected_score"), 

16 [ 

17 ( 

18 [ 

19 "--config", 

20 "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc", 

21 "--trial_config_repeat_count", 

22 "5", 

23 "--mock_env_seed", 

24 "-1", # Deterministic Mock Environment. 

25 ], 

26 67.40329, 

27 ), 

28 ( 

29 [ 

30 "--config", 

31 "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc", 

32 "--trial_config_repeat_count", 

33 "3", 

34 "--max-suggestions", 

35 "3", 

36 "--mock_env_seed", 

37 "42", # Noisy Mock Environment. 

38 ], 

39 64.53897, 

40 ), 

41 ( 

42 [ 

43 "--config", 

44 "mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc", 

45 "--globals", 

46 "experiment_test_local.jsonc", 

47 "--tunable_values", 

48 "tunable-values/tunable-values-local.jsonc", 

49 ], 

50 123.4, 

51 ), 

52 ( 

53 [ 

54 "--config", 

55 "mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-opt.jsonc", 

56 "--globals", 

57 "experiment_test_local.jsonc", 

58 "--max-suggestions", 

59 "3", 

60 ], 

61 123.4, 

62 ), 

63 ], 

64) 

65@pytest.mark.filterwarnings( 

66 "ignore:.*(Configuration.*was already registered).*:UserWarning:.*flaml_optimizer.*:0" 

67) 

68def test_main_bench(argv: List[str], expected_score: float) -> None: 

69 """Run mlos_bench optimization loop with given config and check the results.""" 

70 (score, _config) = _main(argv) 

71 assert score is not None 

72 assert pytest.approx(score["score"], 1e-5) == expected_score