Coverage for mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_smac_test.py: 98%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for mock mlos_bench optimizer. 

7""" 

8import os 

9import sys 

10import shutil 

11 

12import pytest 

13 

14from mlos_bench.util import path_join 

15from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

16from mlos_bench.tunables.tunable_groups import TunableGroups 

17 

18from mlos_bench.tests import SEED 

19 

20from mlos_core.optimizers.bayesian_optimizers.smac_optimizer import SmacOptimizer 

21 

22_OUTPUT_DIR_PATH_BASE = r'c:/temp' if sys.platform == 'win32' else '/tmp/' 

23_OUTPUT_DIR = '_test_output_dir' # Will be deleted after the test. 

24 

25 

26def test_init_mlos_core_smac_opt_bad_trial_count(tunable_groups: TunableGroups) -> None: 

27 """ 

28 Test invalid max_trials initialization of mlos_core SMAC optimizer. 

29 """ 

30 test_opt_config = { 

31 'optimizer_type': 'SMAC', 

32 'max_trials': 10, 

33 'max_suggestions': 11, 

34 'seed': SEED, 

35 } 

36 with pytest.raises(AssertionError): 

37 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

38 assert opt is None 

39 

40 

41def test_init_mlos_core_smac_opt_max_trials(tunable_groups: TunableGroups) -> None: 

42 """ 

43 Test max_trials initialization of mlos_core SMAC optimizer. 

44 """ 

45 test_opt_config = { 

46 'optimizer_type': 'SMAC', 

47 'max_suggestions': 123, 

48 'seed': SEED, 

49 } 

50 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

51 # pylint: disable=protected-access 

52 assert isinstance(opt._opt, SmacOptimizer) 

53 assert opt._opt.base_optimizer.scenario.n_trials == test_opt_config['max_suggestions'] 

54 

55 

56def test_init_mlos_core_smac_absolute_output_directory(tunable_groups: TunableGroups) -> None: 

57 """ 

58 Test absolute path output directory initialization of mlos_core SMAC optimizer. 

59 """ 

60 output_dir = path_join(_OUTPUT_DIR_PATH_BASE, _OUTPUT_DIR) 

61 test_opt_config = { 

62 'optimizer_type': 'SMAC', 

63 'output_directory': output_dir, 

64 'seed': SEED, 

65 } 

66 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

67 assert isinstance(opt, MlosCoreOptimizer) 

68 # pylint: disable=protected-access 

69 assert isinstance(opt._opt, SmacOptimizer) 

70 # Final portions of the path are generated by SMAC when run_name is not specified. 

71 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

72 str(test_opt_config['output_directory'])) 

73 shutil.rmtree(output_dir) 

74 

75 

76def test_init_mlos_core_smac_relative_output_directory(tunable_groups: TunableGroups) -> None: 

77 """ 

78 Test relative path output directory initialization of mlos_core SMAC optimizer. 

79 """ 

80 test_opt_config = { 

81 'optimizer_type': 'SMAC', 

82 'output_directory': _OUTPUT_DIR, 

83 'seed': SEED, 

84 } 

85 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

86 assert isinstance(opt, MlosCoreOptimizer) 

87 # pylint: disable=protected-access 

88 assert isinstance(opt._opt, SmacOptimizer) 

89 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

90 path_join(os.getcwd(), str(test_opt_config['output_directory']))) 

91 shutil.rmtree(_OUTPUT_DIR) 

92 

93 

94def test_init_mlos_core_smac_relative_output_directory_with_run_name(tunable_groups: TunableGroups) -> None: 

95 """ 

96 Test relative path output directory initialization of mlos_core SMAC optimizer. 

97 """ 

98 test_opt_config = { 

99 'optimizer_type': 'SMAC', 

100 'output_directory': _OUTPUT_DIR, 

101 'run_name': 'test_run', 

102 'seed': SEED, 

103 } 

104 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

105 assert isinstance(opt, MlosCoreOptimizer) 

106 # pylint: disable=protected-access 

107 assert isinstance(opt._opt, SmacOptimizer) 

108 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

109 path_join(os.getcwd(), str(test_opt_config['output_directory']), str(test_opt_config['run_name']))) 

110 shutil.rmtree(_OUTPUT_DIR) 

111 

112 

113def test_init_mlos_core_smac_relative_output_directory_with_experiment_id(tunable_groups: TunableGroups) -> None: 

114 """ 

115 Test relative path output directory initialization of mlos_core SMAC optimizer. 

116 """ 

117 test_opt_config = { 

118 'optimizer_type': 'SMAC', 

119 'output_directory': _OUTPUT_DIR, 

120 'seed': SEED, 

121 } 

122 global_config = { 

123 'experiment_id': 'experiment_id', 

124 } 

125 opt = MlosCoreOptimizer(tunable_groups, test_opt_config, global_config) 

126 assert isinstance(opt, MlosCoreOptimizer) 

127 # pylint: disable=protected-access 

128 assert isinstance(opt._opt, SmacOptimizer) 

129 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

130 path_join(os.getcwd(), str(test_opt_config['output_directory']), global_config['experiment_id'])) 

131 shutil.rmtree(_OUTPUT_DIR) 

132 

133 

134def test_init_mlos_core_smac_temp_output_directory(tunable_groups: TunableGroups) -> None: 

135 """ 

136 Test random output directory initialization of mlos_core SMAC optimizer. 

137 """ 

138 test_opt_config = { 

139 'optimizer_type': 'SMAC', 

140 'output_directory': None, 

141 'seed': SEED, 

142 } 

143 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

144 assert isinstance(opt, MlosCoreOptimizer) 

145 # pylint: disable=protected-access 

146 assert isinstance(opt._opt, SmacOptimizer) 

147 assert opt._opt.base_optimizer.scenario.output_directory is not None