Coverage for mlos_bench/mlos_bench/run.py: 87%

1#!/usr/bin/env python3 

2# 

3# Copyright (c) Microsoft Corporation. 

4# Licensed under the MIT License. 

5# 

6""" 

7mlos_bench main optimization loop and benchmark runner CLI. 

8 

9Note: this script is also available as a CLI tool via ``pip`` under the name ``mlos_bench``. 

10 

11See the current ``--help`` `output for details <../../../mlos_bench.run.usage.html>`_. 

12 

13See Also 

14-------- 

15mlos_bench.config : documentation on the configuration system. 

16mlos_bench.launcher.Launcher : class is responsible for processing the CLI args. 

17""" 

18 

19import logging 

20import sys 

21from typing import Dict, List, Optional, Tuple 

22 

23import numpy as np 

24 

25from mlos_bench.launcher import Launcher 

26from mlos_bench.tunables.tunable_groups import TunableGroups 

27 

28_LOG = logging.getLogger(__name__) 

29 

30 

31def _sanity_check_results(launcher: Launcher) -> None: 

32 """Do some sanity checking on the results and throw an exception if it looks like 

33 something went wrong. 

34 """ 

35 basic_err_msg = "Check configuration, scripts, and logs for details." 

36 

37 # Check if the scheduler has any trials. 

38 if not launcher.scheduler.trial_count: 

39 raise RuntimeError(f"No trials were run. {basic_err_msg}") 

40 

41 # Check if the scheduler ran the expected number of trials. 

42 expected_trial_count = min( 

43 launcher.scheduler.max_trials if launcher.scheduler.max_trials > 0 else np.inf, 

44 launcher.scheduler.trial_config_repeat_count * launcher.optimizer.max_suggestions, 

45 ) 

46 if launcher.scheduler.trial_count < expected_trial_count: 

47 raise RuntimeError( 

48 f"Expected {expected_trial_count} trials, " 

49 f"but only {launcher.scheduler.trial_count} were run. {basic_err_msg}" 

50 ) 

51 

52 # Check to see if "too many" trials seem to have failed (#523). 

53 unsuccessful_trials = [t for t in launcher.scheduler.ran_trials if not t.status.is_succeeded()] 

54 if len(unsuccessful_trials) > 0.2 * launcher.scheduler.trial_count: 

55 raise RuntimeWarning( 

56 "Too many trials failed: " 

57 f"{len(unsuccessful_trials)} out of {launcher.scheduler.trial_count}. " 

58 f"{basic_err_msg}" 

59 ) 

60 

61 

62def _main( 

63 argv: Optional[List[str]] = None, 

64) -> Tuple[Optional[Dict[str, float]], Optional[TunableGroups]]: 

65 launcher = Launcher("mlos_bench", "Systems autotuning and benchmarking tool", argv=argv) 

66 

67 with launcher.scheduler as scheduler_context: 

68 scheduler_context.start() 

69 scheduler_context.teardown() 

70 

71 _sanity_check_results(launcher) 

72 

73 (score, _config) = result = launcher.scheduler.get_best_observation() 

74 # NOTE: This log line is used in test_launch_main_app_* unit tests: 

75 _LOG.info("Final score: %s", score) 

76 return result 

77 

78 

79def _shell_main( 

80 argv: Optional[List[str]] = None, 

81) -> int: 

82 (best_score, best_config) = _main(argv) 

83 # Exit zero if it looks like the overall operation was successful. 

84 # TODO: Improve this sanity check to be more robust. 

85 if ( 

86 best_score 

87 and best_config 

88 and all(isinstance(score_value, float) for score_value in best_score.values()) 

89 ): 

90 return 0 

91 else: 

92 raise ValueError(f"Unexpected result: {best_score=}, {best_config=}") 

93 

94 

95if __name__ == "__main__": 

96 sys.exit(_shell_main())