Coverage for mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Base interface for accessing the stored benchmark config trial group data. 

7 

8Since a single config may be used by multiple trials, we can group them together for 

9easier analysis. 

10 

11See Also 

12-------- 

13:py:mod:`mlos_bench.storage` : The base storage module for mlos_bench, which 

14 includes some basic examples in the documentation. 

15""" 

16 

17from abc import ABCMeta, abstractmethod 

18from typing import TYPE_CHECKING, Any, Dict, Optional 

19 

20import pandas 

21 

22from mlos_bench.storage.base_tunable_config_data import TunableConfigData 

23 

24if TYPE_CHECKING: 

25 from mlos_bench.storage.base_trial_data import TrialData 

26 

27 

28class TunableConfigTrialGroupData(metaclass=ABCMeta): 

29 """ 

30 Base interface for accessing the stored experiment benchmark tunable config trial 

31 group data. 

32 

33 A (tunable) config is used to define an instance of values for a set of tunable 

34 parameters for a given experiment and can be used by one or more trial instances 

35 (e.g., for repeats), which we call a (tunable) config trial group. 

36 """ 

37 

38 def __init__( 

39 self, 

40 *, 

41 experiment_id: str, 

42 tunable_config_id: int, 

43 tunable_config_trial_group_id: Optional[int] = None, 

44 ): 

45 self._experiment_id = experiment_id 

46 self._tunable_config_id = tunable_config_id 

47 # can be lazily initialized as necessary: 

48 self._tunable_config_trial_group_id: Optional[int] = tunable_config_trial_group_id 

49 

50 @property 

51 def experiment_id(self) -> str: 

52 """ID of the experiment.""" 

53 return self._experiment_id 

54 

55 @property 

56 def tunable_config_id(self) -> int: 

57 """ID of the config.""" 

58 return self._tunable_config_id 

59 

60 @abstractmethod 

61 def _get_tunable_config_trial_group_id(self) -> int: 

62 """Retrieve the trial's config_trial_group_id from the storage.""" 

63 raise NotImplementedError("subclass must implement") 

64 

65 @property 

66 def tunable_config_trial_group_id(self) -> int: 

67 """ 

68 The unique ID (within this experiment) of the (tunable) config trial group. 

69 

70 This is a unique identifier for all trials in this experiment using the given 

71 config_id, and typically defined as the the minimum trial_id for the given 

72 config_id. 

73 """ 

74 if self._tunable_config_trial_group_id is None: 

75 self._tunable_config_trial_group_id = self._get_tunable_config_trial_group_id() 

76 assert self._tunable_config_trial_group_id is not None 

77 return self._tunable_config_trial_group_id 

78 

79 def __repr__(self) -> str: 

80 return f"TunableConfigTrialGroup :: {self._experiment_id} cid:{self.tunable_config_id}" 

81 

82 def __eq__(self, other: Any) -> bool: 

83 if not isinstance(other, self.__class__): 

84 return False 

85 return ( 

86 self._tunable_config_id == other._tunable_config_id 

87 and self._experiment_id == other._experiment_id 

88 ) 

89 

90 @property 

91 @abstractmethod 

92 def tunable_config(self) -> TunableConfigData: 

93 """ 

94 Retrieve the (tunable) config data for this (tunable) config trial group from 

95 the storage. 

96 

97 Returns 

98 ------- 

99 TunableConfigData 

100 """ 

101 

102 @property 

103 @abstractmethod 

104 def trials(self) -> Dict[int, "TrialData"]: 

105 """ 

106 Retrieve the trials' data for this (tunable) config trial group from the 

107 storage. 

108 

109 Returns 

110 ------- 

111 trials : Dict[int, TrialData] 

112 A dictionary of the trials' data, keyed by trial id. 

113 """ 

114 

115 @property 

116 @abstractmethod 

117 def results_df(self) -> pandas.DataFrame: 

118 """ 

119 Retrieve all results for this (tunable) config trial group as a single 

120 DataFrame. 

121 

122 Returns 

123 ------- 

124 results : pandas.DataFrame 

125 A DataFrame with configurations and results from all trials of the experiment. 

126 Has columns [trial_id, config_id, ts_start, ts_end, status] 

127 followed by tunable config parameters (prefixed with "config.") and 

128 trial results (prefixed with "result."). The latter can be NULLs if the 

129 trial was not successful. 

130 

131 See Also 

132 -------- 

133 :py:attr:`mlos_bench.storage.base_experiment_data.ExperimentData.results_df` 

134 """