Coverage for mlos_bench/mlos_bench/storage/base_trial_data.py: 98%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Base interface for accessing the stored benchmark trial data. 

7 

8A single trial is a single run of an experiment with a given configuration (e.g., set of 

9tunable parameters). 

10 

11See Also 

12-------- 

13:py:mod:`mlos_bench.storage` : The base storage module for mlos_bench, which 

14 includes some basic examples in the documentation. 

15""" 

16from abc import ABCMeta, abstractmethod 

17from datetime import datetime 

18from typing import TYPE_CHECKING, Any, Dict, Optional 

19 

20import pandas 

21from pytz import UTC 

22 

23from mlos_bench.environments.status import Status 

24from mlos_bench.storage.base_tunable_config_data import TunableConfigData 

25from mlos_bench.storage.util import kv_df_to_dict 

26from mlos_bench.tunables.tunable import TunableValue 

27 

28if TYPE_CHECKING: 

29 from mlos_bench.storage.base_tunable_config_trial_group_data import ( 

30 TunableConfigTrialGroupData, 

31 ) 

32 

33 

34class TrialData(metaclass=ABCMeta): 

35 """ 

36 Base interface for accessing the stored experiment benchmark trial data. 

37 

38 A trial is a single run of an experiment with a given configuration (e.g., set of 

39 tunable parameters). 

40 """ 

41 

42 def __init__( # pylint: disable=too-many-arguments 

43 self, 

44 *, 

45 experiment_id: str, 

46 trial_id: int, 

47 tunable_config_id: int, 

48 ts_start: datetime, 

49 ts_end: Optional[datetime], 

50 status: Status, 

51 ): 

52 self._experiment_id = experiment_id 

53 self._trial_id = trial_id 

54 self._tunable_config_id = tunable_config_id 

55 assert ts_start.tzinfo == UTC, "ts_start must be in UTC" 

56 assert ts_end is None or ts_end.tzinfo == UTC, "ts_end must be in UTC if not None" 

57 self._ts_start = ts_start 

58 self._ts_end = ts_end 

59 self._status = status 

60 

61 def __repr__(self) -> str: 

62 return ( 

63 f"Trial :: {self._experiment_id}:{self._trial_id} " 

64 f"cid:{self._tunable_config_id} {self._status.name}" 

65 ) 

66 

67 def __eq__(self, other: Any) -> bool: 

68 if not isinstance(other, self.__class__): 

69 return False 

70 return self._experiment_id == other._experiment_id and self._trial_id == other._trial_id 

71 

72 @property 

73 def experiment_id(self) -> str: 

74 """ID of the experiment this trial belongs to.""" 

75 return self._experiment_id 

76 

77 @property 

78 def trial_id(self) -> int: 

79 """ID of the trial.""" 

80 return self._trial_id 

81 

82 @property 

83 def ts_start(self) -> datetime: 

84 """Start timestamp of the trial (UTC).""" 

85 return self._ts_start 

86 

87 @property 

88 def ts_end(self) -> Optional[datetime]: 

89 """End timestamp of the trial (UTC).""" 

90 return self._ts_end 

91 

92 @property 

93 def status(self) -> Status: 

94 """Status of the trial.""" 

95 return self._status 

96 

97 @property 

98 def tunable_config_id(self) -> int: 

99 """ID of the (tunable) configuration of the trial.""" 

100 return self._tunable_config_id 

101 

102 @property 

103 @abstractmethod 

104 def tunable_config(self) -> TunableConfigData: 

105 """ 

106 Retrieve the trials' tunable configuration data from the storage. 

107 

108 Note: this corresponds to the Trial object's "tunables" property. 

109 

110 Returns 

111 ------- 

112 tunable_config : TunableConfigData 

113 A TunableConfigData object. 

114 """ 

115 

116 @property 

117 @abstractmethod 

118 def tunable_config_trial_group(self) -> "TunableConfigTrialGroupData": 

119 """Retrieve the trial's (tunable) config trial group data from the storage.""" 

120 

121 @property 

122 @abstractmethod 

123 def results_df(self) -> pandas.DataFrame: 

124 """ 

125 Retrieve the trials' results from the storage. 

126 

127 Returns 

128 ------- 

129 results : pandas.DataFrame 

130 A dataframe with the trial results. 

131 It has two `str` columns, "metric" and "value". 

132 If the trial status is not SUCCEEDED, the dataframe is empty. 

133 """ 

134 

135 @property 

136 def results_dict(self) -> Dict[str, Optional[TunableValue]]: 

137 """ 

138 Retrieve the trials' results from the storage as a dict. 

139 

140 Returns 

141 ------- 

142 results : dict 

143 """ 

144 return kv_df_to_dict(self.results_df) 

145 

146 @property 

147 @abstractmethod 

148 def telemetry_df(self) -> pandas.DataFrame: 

149 """ 

150 Retrieve the trials' telemetry from the storage as a dataframe. 

151 

152 Returns 

153 ------- 

154 config : pandas.DataFrame 

155 A dataframe with the trial telemetry, if there is any. 

156 It has one `datetime` column, "ts", and two `str` columns, "metric" and "value". 

157 If the trial status is not SUCCEEDED, or there is no telemetry data, 

158 the dataframe is empty. 

159 """ 

160 

161 @property 

162 @abstractmethod 

163 def metadata_df(self) -> pandas.DataFrame: 

164 """ 

165 Retrieve the trials' metadata parameters as a dataframe. 

166 

167 Note: this corresponds to the Trial object's "config" property. 

168 

169 Returns 

170 ------- 

171 metadata : pandas.DataFrame 

172 An optional dataframe with the metadata associated with the trial. 

173 It has two `str` columns, "parameter" and "value". 

174 Returns an empty dataframe if there is no metadata. 

175 """ 

176 

177 @property 

178 def metadata_dict(self) -> dict: 

179 """ 

180 Retrieve the trials' metadata parameters as a dict. 

181 

182 Note: this corresponds to the Trial object's "config" property. 

183 

184 Returns 

185 ------- 

186 metadata : dict 

187 """ 

188 return kv_df_to_dict(self.metadata_df)