Coverage for mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Toy optimization loop to test the optimizers on mock benchmark environment. 

7""" 

8 

9from typing import Tuple 

10 

11import logging 

12 

13import pytest 

14 

15from mlos_core.util import config_to_dataframe 

16from mlos_core.optimizers.bayesian_optimizers.smac_optimizer import SmacOptimizer 

17from mlos_bench.optimizers.convert_configspace import tunable_values_to_configuration 

18 

19from mlos_bench.environments.base_environment import Environment 

20from mlos_bench.environments.mock_env import MockEnv 

21from mlos_bench.tunables.tunable_groups import TunableGroups 

22from mlos_bench.optimizers.base_optimizer import Optimizer 

23from mlos_bench.optimizers.mock_optimizer import MockOptimizer 

24from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

25 

26 

27# For debugging purposes output some warnings which are captured with failed tests. 

28DEBUG = True 

29logger = logging.debug 

30if DEBUG: 

31 logger = logging.warning 

32 

33 

34def _optimize(env: Environment, opt: Optimizer) -> Tuple[float, TunableGroups]: 

35 """ 

36 Toy optimization loop. 

37 """ 

38 assert opt.not_converged() 

39 

40 while opt.not_converged(): 

41 

42 with env as env_context: 

43 

44 tunables = opt.suggest() 

45 

46 logger("tunables: %s", str(tunables)) 

47 # pylint: disable=protected-access 

48 if isinstance(opt, MlosCoreOptimizer) and isinstance(opt._opt, SmacOptimizer): 

49 config = tunable_values_to_configuration(tunables) 

50 config_df = config_to_dataframe(config) 

51 logger("config: %s", str(config)) 

52 try: 

53 logger("prediction: %s", opt._opt.surrogate_predict(config_df)) 

54 except RuntimeError: 

55 pass 

56 

57 assert env_context.setup(tunables) 

58 

59 (status, _ts, output) = env_context.run() 

60 assert status.is_succeeded() 

61 assert output is not None 

62 score = output['score'] 

63 assert isinstance(score, float) 

64 assert 60 <= score <= 120 

65 logger("score: %s", str(score)) 

66 

67 opt.register(tunables, status, score) 

68 

69 (best_score, best_tunables) = opt.get_best_observation() 

70 assert isinstance(best_score, float) and isinstance(best_tunables, TunableGroups) 

71 return (best_score, best_tunables) 

72 

73 

74def test_mock_optimization_loop(mock_env_no_noise: MockEnv, 

75 mock_opt: MockOptimizer) -> None: 

76 """ 

77 Toy optimization loop with mock environment and optimizer. 

78 """ 

79 (score, tunables) = _optimize(mock_env_no_noise, mock_opt) 

80 assert score == pytest.approx(64.9, 0.01) 

81 assert tunables.get_param_values() == { 

82 "vmSize": "Standard_B2ms", 

83 "idle": "halt", 

84 "kernel_sched_migration_cost_ns": 117026, 

85 "kernel_sched_latency_ns": 149827706, 

86 } 

87 

88 

89def test_mock_optimization_loop_no_defaults(mock_env_no_noise: MockEnv, 

90 mock_opt_no_defaults: MockOptimizer) -> None: 

91 """ 

92 Toy optimization loop with mock environment and optimizer. 

93 """ 

94 (score, tunables) = _optimize(mock_env_no_noise, mock_opt_no_defaults) 

95 assert score == pytest.approx(60.97, 0.01) 

96 assert tunables.get_param_values() == { 

97 "vmSize": "Standard_B2s", 

98 "idle": "halt", 

99 "kernel_sched_migration_cost_ns": 49123, 

100 "kernel_sched_latency_ns": 234760738, 

101 } 

102 

103 

104def test_flaml_optimization_loop(mock_env_no_noise: MockEnv, 

105 flaml_opt: MlosCoreOptimizer) -> None: 

106 """ 

107 Toy optimization loop with mock environment and FLAML optimizer. 

108 """ 

109 (score, tunables) = _optimize(mock_env_no_noise, flaml_opt) 

110 assert score == pytest.approx(60.15, 0.01) 

111 assert tunables.get_param_values() == { 

112 "vmSize": "Standard_B2s", 

113 "idle": "halt", 

114 "kernel_sched_migration_cost_ns": -1, 

115 "kernel_sched_latency_ns": 13718105, 

116 } 

117 

118 

119# @pytest.mark.skip(reason="SMAC is not deterministic") 

120def test_smac_optimization_loop(mock_env_no_noise: MockEnv, 

121 smac_opt: MlosCoreOptimizer) -> None: 

122 """ 

123 Toy optimization loop with mock environment and SMAC optimizer. 

124 """ 

125 (score, tunables) = _optimize(mock_env_no_noise, smac_opt) 

126 expected_score = 70.33 

127 expected_tunable_values = { 

128 "vmSize": "Standard_B2s", 

129 "idle": "mwait", 

130 "kernel_sched_migration_cost_ns": 297669, 

131 "kernel_sched_latency_ns": 290365137, 

132 } 

133 assert score == pytest.approx(expected_score, 0.01) 

134 assert tunables.get_param_values() == expected_tunable_values