Coverage for mlos_bench/mlos_bench/tests/environments/mock_env_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for mock benchmark environment. 

7""" 

8import pytest 

9 

10from mlos_bench.environments.mock_env import MockEnv 

11from mlos_bench.tunables.tunable_groups import TunableGroups 

12 

13 

14def test_mock_env_default(mock_env: MockEnv, tunable_groups: TunableGroups) -> None: 

15 """ 

16 Check the default values of the mock environment. 

17 """ 

18 with mock_env as env_context: 

19 assert env_context.setup(tunable_groups) 

20 (status, _ts, data) = env_context.run() 

21 assert status.is_succeeded() 

22 assert data is not None 

23 assert data["score"] == pytest.approx(73.97, 0.01) 

24 # Second time, results should differ because of the noise. 

25 (status, _ts, data) = env_context.run() 

26 assert status.is_succeeded() 

27 assert data is not None 

28 assert data["score"] == pytest.approx(72.92, 0.01) 

29 

30 

31def test_mock_env_no_noise(mock_env_no_noise: MockEnv, tunable_groups: TunableGroups) -> None: 

32 """ 

33 Check the default values of the mock environment. 

34 """ 

35 with mock_env_no_noise as env_context: 

36 assert env_context.setup(tunable_groups) 

37 for _ in range(10): 

38 # Noise-free results should be the same every time. 

39 (status, _ts, data) = env_context.run() 

40 assert status.is_succeeded() 

41 assert data is not None 

42 assert data["score"] == pytest.approx(75.0, 0.01) 

43 

44 

45@pytest.mark.parametrize(('tunable_values', 'expected_score'), [ 

46 ({ 

47 "vmSize": "Standard_B2ms", 

48 "idle": "halt", 

49 "kernel_sched_migration_cost_ns": 250000 

50 }, 66.4), 

51 ({ 

52 "vmSize": "Standard_B4ms", 

53 "idle": "halt", 

54 "kernel_sched_migration_cost_ns": 40000 

55 }, 74.06), 

56]) 

57def test_mock_env_assign(mock_env: MockEnv, tunable_groups: TunableGroups, 

58 tunable_values: dict, expected_score: float) -> None: 

59 """ 

60 Check the benchmark values of the mock environment after the assignment. 

61 """ 

62 with mock_env as env_context: 

63 tunable_groups.assign(tunable_values) 

64 assert env_context.setup(tunable_groups) 

65 (status, _ts, data) = env_context.run() 

66 assert status.is_succeeded() 

67 assert data is not None 

68 assert data["score"] == pytest.approx(expected_score, 0.01) 

69 

70 

71@pytest.mark.parametrize(('tunable_values', 'expected_score'), [ 

72 ({ 

73 "vmSize": "Standard_B2ms", 

74 "idle": "halt", 

75 "kernel_sched_migration_cost_ns": 250000 

76 }, 67.5), 

77 ({ 

78 "vmSize": "Standard_B4ms", 

79 "idle": "halt", 

80 "kernel_sched_migration_cost_ns": 40000 

81 }, 75.1), 

82]) 

83def test_mock_env_no_noise_assign(mock_env_no_noise: MockEnv, 

84 tunable_groups: TunableGroups, 

85 tunable_values: dict, expected_score: float) -> None: 

86 """ 

87 Check the benchmark values of the noiseless mock environment after the assignment. 

88 """ 

89 with mock_env_no_noise as env_context: 

90 tunable_groups.assign(tunable_values) 

91 assert env_context.setup(tunable_groups) 

92 for _ in range(10): 

93 # Noise-free environment should produce the same results every time. 

94 (status, _ts, data) = env_context.run() 

95 assert status.is_succeeded() 

96 assert data is not None 

97 assert data["score"] == pytest.approx(expected_score, 0.01)