Coverage for mlos_bench/mlos_bench/tests/launcher_parse_args_test.py: 98%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests to check the Launcher CLI arg parsing 

7See Also: test_load_cli_config_examples.py 

8""" 

9 

10import os 

11import sys 

12from getpass import getuser 

13from typing import List 

14 

15import pytest 

16 

17from mlos_bench.launcher import Launcher 

18from mlos_bench.optimizers import OneShotOptimizer, MlosCoreOptimizer 

19from mlos_bench.os_environ import environ 

20from mlos_bench.config.schemas import ConfigSchema 

21from mlos_bench.util import path_join 

22from mlos_bench.schedulers import SyncScheduler 

23from mlos_bench.services.types import ( 

24 SupportsAuth, 

25 SupportsConfigLoading, 

26 SupportsFileShareOps, 

27 SupportsLocalExec, 

28 SupportsRemoteExec, 

29) 

30from mlos_bench.tests import check_class_name 

31 

32if sys.version_info < (3, 10): 

33 from importlib_resources import files 

34else: 

35 from importlib.resources import files 

36 

37# pylint: disable=redefined-outer-name 

38 

39 

40@pytest.fixture 

41def config_paths() -> List[str]: 

42 """ 

43 Returns a list of config paths. 

44 

45 Returns 

46 ------- 

47 List[str] 

48 """ 

49 return [ 

50 path_join(os.getcwd(), abs_path=True), 

51 str(files('mlos_bench.config')), 

52 str(files('mlos_bench.tests.config')), 

53 ] 

54 

55 

56def test_launcher_args_parse_1(config_paths: List[str]) -> None: 

57 """ 

58 Test that using multiple --globals arguments works and that multiple space 

59 separated options to --config-paths works. 

60 Check $var expansion and Environment loading. 

61 """ 

62 # The VSCode pytest wrapper actually starts in a different directory before 

63 # changing into the code directory, but doesn't update the PWD environment 

64 # variable so we use a separate variable. 

65 # See global_test_config.jsonc for more details. 

66 environ["CUSTOM_PATH_FROM_ENV"] = os.getcwd() 

67 if sys.platform == 'win32': 

68 # Some env tweaks for platform compatibility. 

69 environ['USER'] = environ['USERNAME'] 

70 

71 # This is part of the minimal required args by the Launcher. 

72 env_conf_path = 'environments/mock/mock_env.jsonc' 

73 cli_args = '--config-paths ' + ' '.join(config_paths) + \ 

74 ' --service services/remote/mock/mock_auth_service.jsonc' + \ 

75 ' --service services/remote/mock/mock_remote_exec_service.jsonc' + \ 

76 ' --scheduler schedulers/sync_scheduler.jsonc' + \ 

77 f' --environment {env_conf_path}' + \ 

78 ' --globals globals/global_test_config.jsonc' + \ 

79 ' --globals globals/global_test_extra_config.jsonc' \ 

80 ' --test_global_value_2 from-args' 

81 launcher = Launcher(description=__name__, argv=cli_args.split()) 

82 # Check that the parent service 

83 assert isinstance(launcher.service, SupportsAuth) 

84 assert isinstance(launcher.service, SupportsConfigLoading) 

85 assert isinstance(launcher.service, SupportsLocalExec) 

86 assert isinstance(launcher.service, SupportsRemoteExec) 

87 # Check that the first --globals file is loaded and $var expansion is handled. 

88 assert launcher.global_config['experiment_id'] == 'MockExperiment' 

89 assert launcher.global_config['testVmName'] == 'MockExperiment-vm' 

90 # Check that secondary expansion also works. 

91 assert launcher.global_config['testVnetName'] == 'MockExperiment-vm-vnet' 

92 # Check that the second --globals file is loaded. 

93 assert launcher.global_config['test_global_value'] == 'from-file' 

94 # Check overriding values in a file from the command line. 

95 assert launcher.global_config['test_global_value_2'] == 'from-args' 

96 # Check that we can expand a $var in a config file that references an environment variable. 

97 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) \ 

98 == path_join(os.getcwd(), "foo", abs_path=True) 

99 assert launcher.global_config["varWithEnvVarRef"] == f'user:{getuser()}' 

100 assert launcher.teardown 

101 # Check that the environment that got loaded looks to be of the right type. 

102 env_config = launcher.config_loader.load_config(env_conf_path, ConfigSchema.ENVIRONMENT) 

103 assert check_class_name(launcher.environment, env_config['class']) 

104 # Check that the optimizer looks right. 

105 assert isinstance(launcher.optimizer, OneShotOptimizer) 

106 # Check that the optimizer got initialized with defaults. 

107 assert launcher.optimizer.tunable_params.is_defaults() 

108 assert launcher.optimizer.max_iterations == 1 # value for OneShotOptimizer 

109 # Check that we pick up the right scheduler config: 

110 assert isinstance(launcher.scheduler, SyncScheduler) 

111 assert launcher.scheduler._trial_config_repeat_count == 3 # pylint: disable=protected-access 

112 assert launcher.scheduler._max_trials == -1 # pylint: disable=protected-access 

113 

114 

115def test_launcher_args_parse_2(config_paths: List[str]) -> None: 

116 """ 

117 Test multiple --config-path instances, --config file vs --arg, --var=val 

118 overrides, $var templates, option args, --random-init, etc. 

119 """ 

120 # The VSCode pytest wrapper actually starts in a different directory before 

121 # changing into the code directory, but doesn't update the PWD environment 

122 # variable so we use a separate variable. 

123 # See global_test_config.jsonc for more details. 

124 environ["CUSTOM_PATH_FROM_ENV"] = os.getcwd() 

125 if sys.platform == 'win32': 

126 # Some env tweaks for platform compatibility. 

127 environ['USER'] = environ['USERNAME'] 

128 

129 config_file = 'cli/test-cli-config.jsonc' 

130 globals_file = 'globals/global_test_config.jsonc' 

131 cli_args = ' '.join([f"--config-path {config_path}" for config_path in config_paths]) + \ 

132 f' --config {config_file}' + \ 

133 ' --service services/remote/mock/mock_auth_service.jsonc' + \ 

134 ' --service services/remote/mock/mock_remote_exec_service.jsonc' + \ 

135 f' --globals {globals_file}' + \ 

136 ' --experiment_id MockeryExperiment' + \ 

137 ' --no-teardown' + \ 

138 ' --random-init' + \ 

139 ' --random-seed 1234' + \ 

140 ' --trial-config-repeat-count 5' + \ 

141 ' --max_trials 200' 

142 launcher = Launcher(description=__name__, argv=cli_args.split()) 

143 # Check that the parent service 

144 assert isinstance(launcher.service, SupportsAuth) 

145 assert isinstance(launcher.service, SupportsConfigLoading) 

146 assert isinstance(launcher.service, SupportsFileShareOps) 

147 assert isinstance(launcher.service, SupportsLocalExec) 

148 assert isinstance(launcher.service, SupportsRemoteExec) 

149 # Check that the --globals file is loaded and $var expansion is handled 

150 # using the value provided on the CLI. 

151 assert launcher.global_config['experiment_id'] == 'MockeryExperiment' 

152 assert launcher.global_config['testVmName'] == 'MockeryExperiment-vm' 

153 # Check that secondary expansion also works. 

154 assert launcher.global_config['testVnetName'] == 'MockeryExperiment-vm-vnet' 

155 # Check that we can expand a $var in a config file that references an environment variable. 

156 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) \ 

157 == path_join(os.getcwd(), "foo", abs_path=True) 

158 assert launcher.global_config["varWithEnvVarRef"] == f'user:{getuser()}' 

159 assert not launcher.teardown 

160 

161 config = launcher.config_loader.load_config(config_file, ConfigSchema.CLI) 

162 assert launcher.config_loader.config_paths == [path_join(path, abs_path=True) for path in config_paths + config['config_path']] 

163 

164 # Check that the environment that got loaded looks to be of the right type. 

165 env_config_file = config['environment'] 

166 env_config = launcher.config_loader.load_config(env_config_file, ConfigSchema.ENVIRONMENT) 

167 assert check_class_name(launcher.environment, env_config['class']) 

168 

169 # Check that the optimizer looks right. 

170 assert isinstance(launcher.optimizer, MlosCoreOptimizer) 

171 opt_config_file = config['optimizer'] 

172 opt_config = launcher.config_loader.load_config(opt_config_file, ConfigSchema.OPTIMIZER) 

173 globals_file_config = launcher.config_loader.load_config(globals_file, ConfigSchema.GLOBALS) 

174 # The actual global_config gets overwritten as a part of processing, so to test 

175 # this we read the original value out of the source files. 

176 orig_max_iters = globals_file_config.get('max_suggestions', opt_config.get('config', {}).get('max_suggestions', 100)) 

177 assert launcher.optimizer.max_iterations \ 

178 == orig_max_iters \ 

179 == launcher.global_config['max_suggestions'] 

180 

181 # Check that the optimizer got initialized with random values instead of the defaults. 

182 # Note: the environment doesn't get updated until suggest() is called to 

183 # return these values in run.py. 

184 assert not launcher.optimizer.tunable_params.is_defaults() 

185 

186 # TODO: Add a check that this flows through and replaces other seed config 

187 # values through the stack. 

188 # See Also: #495 

189 

190 # Check that CLI parameter overrides JSON config: 

191 assert isinstance(launcher.scheduler, SyncScheduler) 

192 assert launcher.scheduler._trial_config_repeat_count == 5 # pylint: disable=protected-access 

193 assert launcher.scheduler._max_trials == 200 # pylint: disable=protected-access 

194 

195 # Check that the value from the file is overridden by the CLI arg. 

196 assert config['random_seed'] == 42 

197 # TODO: This isn't actually respected yet because the `--random-init` only 

198 # applies to a temporary Optimizer used to populate the initial values via 

199 # random sampling. 

200 # assert launcher.optimizer.seed == 1234 

201 

202 

203if __name__ == '__main__': 

204 pytest.main([__file__, "-n1"])