Coverage for mlos_bench/mlos_bench/tests/config/cli/test_load_cli_config_examples.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Tests for loading storage config examples.""" 

6 

7import logging 

8import sys 

9from typing import List 

10 

11import pytest 

12 

13from mlos_bench.config.schemas import ConfigSchema 

14from mlos_bench.environments import Environment 

15from mlos_bench.launcher import Launcher 

16from mlos_bench.optimizers import Optimizer 

17from mlos_bench.schedulers import Scheduler 

18from mlos_bench.services.config_persistence import ConfigPersistenceService 

19from mlos_bench.storage import Storage 

20from mlos_bench.tests import check_class_name 

21from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples 

22from mlos_bench.util import path_join 

23 

24if sys.version_info < (3, 10): 

25 from importlib_resources import files 

26else: 

27 from importlib.resources import files 

28 

29 

30_LOG = logging.getLogger(__name__) 

31_LOG.setLevel(logging.DEBUG) 

32 

33 

34# Get the set of configs to test. 

35CONFIG_TYPE = "cli" 

36 

37 

38def filter_configs(configs_to_filter: List[str]) -> List[str]: 

39 """If necessary, filter out json files that aren't for the module we're testing.""" 

40 return configs_to_filter 

41 

42 

43configs = [ 

44 *locate_config_examples( 

45 ConfigPersistenceService.BUILTIN_CONFIG_PATH, 

46 CONFIG_TYPE, 

47 filter_configs, 

48 ), 

49 *locate_config_examples( 

50 BUILTIN_TEST_CONFIG_PATH, 

51 CONFIG_TYPE, 

52 filter_configs, 

53 ), 

54] 

55assert configs 

56 

57 

58@pytest.mark.skip(reason="Use full Launcher test (below) instead now.") 

59@pytest.mark.parametrize("config_path", configs) 

60def test_load_cli_config_examples( 

61 config_loader_service: ConfigPersistenceService, 

62 config_path: str, 

63) -> None: # pragma: no cover 

64 """Tests loading a config example.""" 

65 # pylint: disable=too-complex 

66 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

67 assert isinstance(config, dict) 

68 

69 if config_paths := config.get("config_path"): 

70 assert isinstance(config_paths, list) 

71 config_paths.reverse() 

72 for path in config_paths: 

73 config_loader_service._config_path.insert(0, path) # pylint: disable=protected-access 

74 

75 # Foreach arg that references another file, see if we can at least load that too. 

76 args_to_skip = { 

77 "config_path", # handled above 

78 "log_file", 

79 "log_level", 

80 "experiment_id", 

81 "trial_id", 

82 "teardown", 

83 } 

84 for arg in config: 

85 if arg in args_to_skip: 

86 continue 

87 

88 if arg == "globals": 

89 for path in config[arg]: 

90 sub_config = config_loader_service.load_config(path, ConfigSchema.GLOBALS) 

91 assert isinstance(sub_config, dict) 

92 elif arg == "environment": 

93 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.ENVIRONMENT) 

94 assert isinstance(sub_config, dict) 

95 elif arg == "optimizer": 

96 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.OPTIMIZER) 

97 assert isinstance(sub_config, dict) 

98 elif arg == "storage": 

99 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.STORAGE) 

100 assert isinstance(sub_config, dict) 

101 elif arg == "tunable_values": 

102 for path in config[arg]: 

103 sub_config = config_loader_service.load_config(path, ConfigSchema.TUNABLE_VALUES) 

104 assert isinstance(sub_config, dict) 

105 else: 

106 raise NotImplementedError(f"Unhandled arg {arg} in config {config_path}") 

107 

108 

109@pytest.mark.parametrize("config_path", configs) 

110def test_load_cli_config_examples_via_launcher( 

111 config_loader_service: ConfigPersistenceService, 

112 config_path: str, 

113) -> None: 

114 """Tests loading a config example via the Launcher.""" 

115 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

116 assert isinstance(config, dict) 

117 

118 # Try to load the CLI config by instantiating a launcher. 

119 # To do this we need to make sure to give it a few extra paths and globals 

120 # to look for for our examples. 

121 cli_args = ( 

122 # pylint: disable=inconsistent-quotes 

123 f"--config {config_path}" 

124 f" --config-path {files('mlos_bench.config')} " 

125 f" --config-path {files('mlos_bench.tests.config')}" 

126 f" --config-path {path_join(str(files('mlos_bench.tests.config')), 'globals')}" 

127 f" --globals {files('mlos_bench.tests.config')}/experiments/experiment_test_config.jsonc" 

128 ) 

129 launcher = Launcher(description=__name__, long_text=config_path, argv=cli_args.split()) 

130 assert launcher 

131 

132 # Check that some parts of that config are loaded. 

133 

134 assert ConfigPersistenceService.BUILTIN_CONFIG_PATH in launcher.config_loader.config_paths 

135 if config_paths := config.get("config_path"): 

136 assert isinstance(config_paths, list) 

137 for path in config_paths: 

138 # Note: Checks that the order is maintained are handled in launcher_parse_args.py 

139 assert any( 

140 config_path.endswith(path) for config_path in launcher.config_loader.config_paths 

141 ), f"Expected {path} to be in {launcher.config_loader.config_paths}" 

142 

143 if "experiment_id" in config: 

144 assert launcher.global_config["experiment_id"] == config["experiment_id"] 

145 if "trial_id" in config: 

146 assert launcher.global_config["trial_id"] == config["trial_id"] 

147 

148 expected_log_level = logging.getLevelName(config.get("log_level", "INFO")) 

149 if isinstance(expected_log_level, int): 

150 expected_log_level = logging.getLevelName(expected_log_level) 

151 current_log_level = logging.getLevelName(logging.root.getEffectiveLevel()) 

152 assert current_log_level == expected_log_level 

153 

154 # TODO: Check that the log_file handler is set correctly. 

155 

156 expected_teardown = config.get("teardown", True) 

157 assert launcher.teardown == expected_teardown 

158 

159 # Note: Testing of "globals" processing handled in launcher_parse_args_test.py 

160 

161 # Instead of just checking that the config is loaded, check that the 

162 # Launcher loaded the expected types as well. 

163 

164 assert isinstance(launcher.environment, Environment) 

165 env_config = launcher.config_loader.load_config( 

166 config["environment"], 

167 ConfigSchema.ENVIRONMENT, 

168 ) 

169 assert check_class_name(launcher.environment, env_config["class"]) 

170 

171 assert isinstance(launcher.optimizer, Optimizer) 

172 if "optimizer" in config: 

173 opt_config = launcher.config_loader.load_config( 

174 config["optimizer"], 

175 ConfigSchema.OPTIMIZER, 

176 ) 

177 assert check_class_name(launcher.optimizer, opt_config["class"]) 

178 

179 assert isinstance(launcher.storage, Storage) 

180 if "storage" in config: 

181 storage_config = launcher.config_loader.load_config( 

182 config["storage"], 

183 ConfigSchema.STORAGE, 

184 ) 

185 assert check_class_name(launcher.storage, storage_config["class"]) 

186 

187 assert isinstance(launcher.scheduler, Scheduler) 

188 if "scheduler" in config: 

189 scheduler_config = launcher.config_loader.load_config( 

190 config["scheduler"], 

191 ConfigSchema.SCHEDULER, 

192 ) 

193 assert check_class_name(launcher.scheduler, scheduler_config["class"]) 

194 

195 # TODO: Check that the launcher assigns the tunables values as expected.