Coverage for mlos_bench/mlos_bench/optimizers/manual_optimizer.py: 95%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Manual config suggester (Optimizer) for mlos_bench that proposes an explicit sequence of 

7configurations. 

8 

9This is useful for testing and validation, as it allows you to run a sequence of 

10configurations in a cyclic fashion. 

11 

12Examples 

13-------- 

14>>> # Load tunables from a JSON string. 

15>>> # Note: normally these would be automatically loaded from the Environment(s)'s 

16>>> # `include_tunables` config parameter. 

17>>> # 

18>>> import json5 as json 

19>>> from mlos_bench.environments.status import Status 

20>>> from mlos_bench.services.config_persistence import ConfigPersistenceService 

21>>> service = ConfigPersistenceService() 

22>>> json_config = ''' 

23... { 

24... "group_1": { 

25... "cost": 1, 

26... "params": { 

27... "colors": { 

28... "type": "categorical", 

29... "values": ["red", "blue", "green"], 

30... "default": "green", 

31... }, 

32... "int_param": { 

33... "type": "int", 

34... "range": [1, 3], 

35... "default": 2, 

36... }, 

37... "float_param": { 

38... "type": "float", 

39... "range": [0, 1], 

40... "default": 0.5, 

41... // Quantize the range into 3 bins 

42... "quantization_bins": 3, 

43... } 

44... } 

45... } 

46... } 

47... ''' 

48>>> tunables = service.load_tunables(jsons=[json_config]) 

49>>> # Check the defaults: 

50>>> tunables.get_param_values() 

51{'colors': 'green', 'int_param': 2, 'float_param': 0.5} 

52 

53>>> # Now create a ManualOptimizer from a JSON config string. 

54>>> optimizer_json_config = ''' 

55... { 

56... "class": "mlos_bench.optimizers.manual_optimizer.ManualOptimizer", 

57... "description": "ManualOptimizer", 

58... "config": { 

59... "max_cycles": 3, 

60... "tunable_values_cycle": [ 

61... {"colors": "red", "int_param": 1, "float_param": 0.0}, 

62... {"colors": "blue", "int_param": 3, "float_param": 1.0}, 

63... // special case: {} - represents the defaults, without 

64... // having to copy them from the tunables JSON 

65... // (which is presumably specified elsewhere) 

66... {}, 

67... ], 

68... } 

69... } 

70... ''' 

71>>> config = json.loads(optimizer_json_config) 

72>>> optimizer = service.build_optimizer( 

73... tunables=tunables, 

74... service=service, 

75... config=config, 

76... ) 

77>>> # Run the optimizer. 

78>>> # Note that the cycle will repeat 3 times, as specified in the config. 

79>>> while optimizer.not_converged(): 

80... suggestion = optimizer.suggest() 

81... print(suggestion.get_param_values()) 

82{'colors': 'red', 'int_param': 1, 'float_param': 0.0} 

83{'colors': 'blue', 'int_param': 3, 'float_param': 1.0} 

84{'colors': 'green', 'int_param': 2, 'float_param': 0.5} 

85{'colors': 'red', 'int_param': 1, 'float_param': 0.0} 

86{'colors': 'blue', 'int_param': 3, 'float_param': 1.0} 

87{'colors': 'green', 'int_param': 2, 'float_param': 0.5} 

88{'colors': 'red', 'int_param': 1, 'float_param': 0.0} 

89{'colors': 'blue', 'int_param': 3, 'float_param': 1.0} 

90{'colors': 'green', 'int_param': 2, 'float_param': 0.5} 

91""" 

92 

93import logging 

94 

95from mlos_bench.optimizers.mock_optimizer import MockOptimizer 

96from mlos_bench.services.base_service import Service 

97from mlos_bench.tunables.tunable_groups import TunableGroups 

98from mlos_bench.tunables.tunable_types import TunableValue 

99 

100_LOG = logging.getLogger(__name__) 

101 

102 

103class ManualOptimizer(MockOptimizer): 

104 """Optimizer that proposes an explicit sequence of tunable values.""" 

105 

106 def __init__( 

107 self, 

108 tunables: TunableGroups, 

109 config: dict, 

110 global_config: dict | None = None, 

111 service: Service | None = None, 

112 ): 

113 super().__init__(tunables, config, global_config, service) 

114 self._tunable_values_cycle: list[dict[str, TunableValue]] = config.get( 

115 "tunable_values_cycle", [] 

116 ) 

117 assert len(self._tunable_values_cycle) > 0, "No tunable values provided." 

118 max_cycles = int(config.get("max_cycles", 1)) 

119 self._max_suggestions = min( 

120 self._max_suggestions, 

121 max_cycles * len(self._tunable_values_cycle), 

122 ) 

123 

124 def suggest(self) -> TunableGroups: 

125 """Always produce the same sequence of explicit suggestions, in a cycle.""" 

126 tunables = super().suggest() 

127 cycle_index = (self._iter - 1) % len(self._tunable_values_cycle) 

128 tunables.assign(self._tunable_values_cycle[cycle_index]) 

129 _LOG.info("Iteration %d :: Suggest: %s", self._iter, tunables) 

130 return tunables 

131 

132 @property 

133 def supports_preload(self) -> bool: 

134 return False