Coverage for mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_smac_test.py: 98%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for mock mlos_bench optimizer.""" 

6import os 

7import shutil 

8import sys 

9 

10import pytest 

11 

12from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

13from mlos_bench.tests import SEED 

14from mlos_bench.tunables.tunable_groups import TunableGroups 

15from mlos_bench.util import path_join 

16from mlos_core.optimizers.bayesian_optimizers.smac_optimizer import SmacOptimizer 

17 

18_OUTPUT_DIR_PATH_BASE = r"c:/temp" if sys.platform == "win32" else "/tmp/" 

19_OUTPUT_DIR = "_test_output_dir" # Will be deleted after the test. 

20 

21 

22def test_init_mlos_core_smac_opt_bad_trial_count(tunable_groups: TunableGroups) -> None: 

23 """Test invalid max_trials initialization of mlos_core SMAC optimizer.""" 

24 test_opt_config = { 

25 "optimizer_type": "SMAC", 

26 "max_trials": 10, 

27 "max_suggestions": 11, 

28 "seed": SEED, 

29 } 

30 with pytest.raises(AssertionError): 

31 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

32 assert opt is None 

33 

34 

35def test_init_mlos_core_smac_opt_max_trials(tunable_groups: TunableGroups) -> None: 

36 """Test max_trials initialization of mlos_core SMAC optimizer.""" 

37 test_opt_config = { 

38 "optimizer_type": "SMAC", 

39 "max_suggestions": 123, 

40 "seed": SEED, 

41 } 

42 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

43 # pylint: disable=protected-access 

44 assert isinstance(opt._opt, SmacOptimizer) 

45 assert opt._opt.base_optimizer.scenario.n_trials == test_opt_config["max_suggestions"] 

46 

47 

48def test_init_mlos_core_smac_absolute_output_directory(tunable_groups: TunableGroups) -> None: 

49 """Test absolute path output directory initialization of mlos_core SMAC 

50 optimizer. 

51 """ 

52 output_dir = path_join(_OUTPUT_DIR_PATH_BASE, _OUTPUT_DIR) 

53 test_opt_config = { 

54 "optimizer_type": "SMAC", 

55 "output_directory": output_dir, 

56 "seed": SEED, 

57 } 

58 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

59 assert isinstance(opt, MlosCoreOptimizer) 

60 # pylint: disable=protected-access 

61 assert isinstance(opt._opt, SmacOptimizer) 

62 # Final portions of the path are generated by SMAC when run_name is not specified. 

63 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

64 str(test_opt_config["output_directory"]) 

65 ) 

66 shutil.rmtree(output_dir) 

67 

68 

69def test_init_mlos_core_smac_relative_output_directory(tunable_groups: TunableGroups) -> None: 

70 """Test relative path output directory initialization of mlos_core SMAC 

71 optimizer. 

72 """ 

73 uid = os.environ.get("PYTEST_XDIST_WORKER", "") 

74 output_dir = _OUTPUT_DIR + "." + uid 

75 test_opt_config = { 

76 "optimizer_type": "SMAC", 

77 "output_directory": output_dir, 

78 "seed": SEED, 

79 } 

80 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

81 assert isinstance(opt, MlosCoreOptimizer) 

82 # pylint: disable=protected-access 

83 assert isinstance(opt._opt, SmacOptimizer) 

84 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

85 path_join(os.getcwd(), str(test_opt_config["output_directory"])) 

86 ) 

87 shutil.rmtree(output_dir) 

88 

89 

90def test_init_mlos_core_smac_relative_output_directory_with_run_name( 

91 tunable_groups: TunableGroups, 

92) -> None: 

93 """Test relative path output directory initialization of mlos_core SMAC 

94 optimizer. 

95 """ 

96 uid = os.environ.get("PYTEST_XDIST_WORKER", "") 

97 output_dir = _OUTPUT_DIR + "." + uid 

98 test_opt_config = { 

99 "optimizer_type": "SMAC", 

100 "output_directory": output_dir, 

101 "run_name": "test_run", 

102 "seed": SEED, 

103 } 

104 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

105 assert isinstance(opt, MlosCoreOptimizer) 

106 # pylint: disable=protected-access 

107 assert isinstance(opt._opt, SmacOptimizer) 

108 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

109 path_join( 

110 os.getcwd(), str(test_opt_config["output_directory"]), str(test_opt_config["run_name"]) 

111 ) 

112 ) 

113 shutil.rmtree(output_dir) 

114 

115 

116def test_init_mlos_core_smac_relative_output_directory_with_experiment_id( 

117 tunable_groups: TunableGroups, 

118) -> None: 

119 """Test relative path output directory initialization of mlos_core SMAC 

120 optimizer. 

121 """ 

122 test_opt_config = { 

123 "optimizer_type": "SMAC", 

124 "output_directory": _OUTPUT_DIR, 

125 "seed": SEED, 

126 } 

127 global_config = { 

128 "experiment_id": "experiment_id", 

129 } 

130 opt = MlosCoreOptimizer(tunable_groups, test_opt_config, global_config) 

131 assert isinstance(opt, MlosCoreOptimizer) 

132 # pylint: disable=protected-access 

133 assert isinstance(opt._opt, SmacOptimizer) 

134 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

135 path_join( 

136 os.getcwd(), 

137 str(test_opt_config["output_directory"]), 

138 global_config["experiment_id"], 

139 ) 

140 ) 

141 shutil.rmtree(_OUTPUT_DIR) 

142 

143 

144def test_init_mlos_core_smac_temp_output_directory(tunable_groups: TunableGroups) -> None: 

145 """Test random output directory initialization of mlos_core SMAC optimizer.""" 

146 test_opt_config = { 

147 "optimizer_type": "SMAC", 

148 "output_directory": None, 

149 "seed": SEED, 

150 } 

151 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

152 assert isinstance(opt, MlosCoreOptimizer) 

153 # pylint: disable=protected-access 

154 assert isinstance(opt._opt, SmacOptimizer) 

155 assert opt._opt.base_optimizer.scenario.output_directory is not None