Coverage for mlos_bench/mlos_bench/tests/config/cli/test_load_cli_config_examples.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Tests for loading storage config examples. 

7""" 

8 

9from typing import List 

10 

11import logging 

12import sys 

13 

14import pytest 

15 

16from mlos_bench.tests import check_class_name 

17from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH 

18 

19from mlos_bench.config.schemas import ConfigSchema 

20from mlos_bench.environments import Environment 

21from mlos_bench.optimizers import Optimizer 

22from mlos_bench.storage import Storage 

23from mlos_bench.schedulers import Scheduler 

24from mlos_bench.services.config_persistence import ConfigPersistenceService 

25from mlos_bench.launcher import Launcher 

26from mlos_bench.util import path_join 

27 

28if sys.version_info < (3, 10): 

29 from importlib_resources import files 

30else: 

31 from importlib.resources import files 

32 

33 

34_LOG = logging.getLogger(__name__) 

35_LOG.setLevel(logging.DEBUG) 

36 

37 

38# Get the set of configs to test. 

39CONFIG_TYPE = "cli" 

40 

41 

42def filter_configs(configs_to_filter: List[str]) -> List[str]: 

43 """If necessary, filter out json files that aren't for the module we're testing.""" 

44 return configs_to_filter 

45 

46 

47configs = [ 

48 *locate_config_examples(ConfigPersistenceService.BUILTIN_CONFIG_PATH, CONFIG_TYPE, filter_configs), 

49 *locate_config_examples(BUILTIN_TEST_CONFIG_PATH, CONFIG_TYPE, filter_configs), 

50] 

51assert configs 

52 

53 

54@pytest.mark.skip(reason="Use full Launcher test (below) instead now.") 

55@pytest.mark.parametrize("config_path", configs) 

56def test_load_cli_config_examples(config_loader_service: ConfigPersistenceService, config_path: str) -> None: # pragma: no cover 

57 """Tests loading a config example.""" 

58 # pylint: disable=too-complex 

59 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

60 assert isinstance(config, dict) 

61 

62 if config_paths := config.get("config_path"): 

63 assert isinstance(config_paths, list) 

64 config_paths.reverse() 

65 for path in config_paths: 

66 config_loader_service._config_path.insert(0, path) # pylint: disable=protected-access 

67 

68 # Foreach arg that references another file, see if we can at least load that too. 

69 args_to_skip = { 

70 "config_path", # handled above 

71 "log_file", 

72 "log_level", 

73 "experiment_id", 

74 "trial_id", 

75 "teardown", 

76 } 

77 for arg in config: 

78 if arg in args_to_skip: 

79 continue 

80 

81 if arg == "globals": 

82 for path in config[arg]: 

83 sub_config = config_loader_service.load_config(path, ConfigSchema.GLOBALS) 

84 assert isinstance(sub_config, dict) 

85 elif arg == "environment": 

86 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.ENVIRONMENT) 

87 assert isinstance(sub_config, dict) 

88 elif arg == "optimizer": 

89 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.OPTIMIZER) 

90 assert isinstance(sub_config, dict) 

91 elif arg == "storage": 

92 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.STORAGE) 

93 assert isinstance(sub_config, dict) 

94 elif arg == "tunable_values": 

95 for path in config[arg]: 

96 sub_config = config_loader_service.load_config(path, ConfigSchema.TUNABLE_VALUES) 

97 assert isinstance(sub_config, dict) 

98 else: 

99 raise NotImplementedError(f"Unhandled arg {arg} in config {config_path}") 

100 

101 

102@pytest.mark.parametrize("config_path", configs) 

103def test_load_cli_config_examples_via_launcher(config_loader_service: ConfigPersistenceService, config_path: str) -> None: 

104 """Tests loading a config example via the Launcher.""" 

105 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

106 assert isinstance(config, dict) 

107 

108 # Try to load the CLI config by instantiating a launcher. 

109 # To do this we need to make sure to give it a few extra paths and globals 

110 # to look for for our examples. 

111 cli_args = f"--config {config_path}" + \ 

112 f" --config-path {files('mlos_bench.config')} --config-path {files('mlos_bench.tests.config')}" + \ 

113 f" --config-path {path_join(str(files('mlos_bench.tests.config')), 'globals')}" + \ 

114 f" --globals {files('mlos_bench.tests.config')}/experiments/experiment_test_config.jsonc" 

115 launcher = Launcher(description=__name__, long_text=config_path, argv=cli_args.split()) 

116 assert launcher 

117 

118 # Check that some parts of that config are loaded. 

119 

120 assert ConfigPersistenceService.BUILTIN_CONFIG_PATH in launcher.config_loader.config_paths 

121 if config_paths := config.get("config_path"): 

122 assert isinstance(config_paths, list) 

123 for path in config_paths: 

124 # Note: Checks that the order is maintained are handled in launcher_parse_args.py 

125 assert any(config_path.endswith(path) for config_path in launcher.config_loader.config_paths), \ 

126 f"Expected {path} to be in {launcher.config_loader.config_paths}" 

127 

128 if 'experiment_id' in config: 

129 assert launcher.global_config['experiment_id'] == config['experiment_id'] 

130 if 'trial_id' in config: 

131 assert launcher.global_config['trial_id'] == config['trial_id'] 

132 

133 expected_log_level = logging.getLevelName(config.get('log_level', "INFO")) 

134 if isinstance(expected_log_level, int): 

135 expected_log_level = logging.getLevelName(expected_log_level) 

136 current_log_level = logging.getLevelName(logging.root.getEffectiveLevel()) 

137 assert current_log_level == expected_log_level 

138 

139 # TODO: Check that the log_file handler is set correctly. 

140 

141 expected_teardown = config.get('teardown', True) 

142 assert launcher.teardown == expected_teardown 

143 

144 # Note: Testing of "globals" processing handled in launcher_parse_args_test.py 

145 

146 # Instead of just checking that the config is loaded, check that the 

147 # Launcher loaded the expected types as well. 

148 

149 assert isinstance(launcher.environment, Environment) 

150 env_config = launcher.config_loader.load_config(config["environment"], ConfigSchema.ENVIRONMENT) 

151 assert check_class_name(launcher.environment, env_config["class"]) 

152 

153 assert isinstance(launcher.optimizer, Optimizer) 

154 if "optimizer" in config: 

155 opt_config = launcher.config_loader.load_config(config["optimizer"], ConfigSchema.OPTIMIZER) 

156 assert check_class_name(launcher.optimizer, opt_config["class"]) 

157 

158 assert isinstance(launcher.storage, Storage) 

159 if "storage" in config: 

160 storage_config = launcher.config_loader.load_config(config["storage"], ConfigSchema.STORAGE) 

161 assert check_class_name(launcher.storage, storage_config["class"]) 

162 

163 assert isinstance(launcher.scheduler, Scheduler) 

164 if "scheduler" in config: 

165 scheduler_config = launcher.config_loader.load_config(config["scheduler"], ConfigSchema.SCHEDULER) 

166 assert check_class_name(launcher.scheduler, scheduler_config["class"]) 

167 

168 # TODO: Check that the launcher assigns the tunables values as expected.