Coverage for mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Base interface for accessing the stored benchmark config trial group data. 

7 

8Since a single config may be used by multiple trials, we can group them together for 

9easier analysis. 

10""" 

11 

12from abc import ABCMeta, abstractmethod 

13from typing import TYPE_CHECKING, Any, Dict, Optional 

14 

15import pandas 

16 

17from mlos_bench.storage.base_tunable_config_data import TunableConfigData 

18 

19if TYPE_CHECKING: 

20 from mlos_bench.storage.base_trial_data import TrialData 

21 

22 

23class TunableConfigTrialGroupData(metaclass=ABCMeta): 

24 """ 

25 Base interface for accessing the stored experiment benchmark tunable config trial 

26 group data. 

27 

28 A (tunable) config is used to define an instance of values for a set of tunable 

29 parameters for a given experiment and can be used by one or more trial instances 

30 (e.g., for repeats), which we call a (tunable) config trial group. 

31 """ 

32 

33 def __init__( 

34 self, 

35 *, 

36 experiment_id: str, 

37 tunable_config_id: int, 

38 tunable_config_trial_group_id: Optional[int] = None, 

39 ): 

40 self._experiment_id = experiment_id 

41 self._tunable_config_id = tunable_config_id 

42 # can be lazily initialized as necessary: 

43 self._tunable_config_trial_group_id: Optional[int] = tunable_config_trial_group_id 

44 

45 @property 

46 def experiment_id(self) -> str: 

47 """ID of the experiment.""" 

48 return self._experiment_id 

49 

50 @property 

51 def tunable_config_id(self) -> int: 

52 """ID of the config.""" 

53 return self._tunable_config_id 

54 

55 @abstractmethod 

56 def _get_tunable_config_trial_group_id(self) -> int: 

57 """Retrieve the trial's config_trial_group_id from the storage.""" 

58 raise NotImplementedError("subclass must implement") 

59 

60 @property 

61 def tunable_config_trial_group_id(self) -> int: 

62 """ 

63 The unique ID (within this experiment) of the (tunable) config trial group. 

64 

65 This is a unique identifier for all trials in this experiment using the given 

66 config_id, and typically defined as the the minimum trial_id for the given 

67 config_id. 

68 """ 

69 if self._tunable_config_trial_group_id is None: 

70 self._tunable_config_trial_group_id = self._get_tunable_config_trial_group_id() 

71 assert self._tunable_config_trial_group_id is not None 

72 return self._tunable_config_trial_group_id 

73 

74 def __repr__(self) -> str: 

75 return f"TunableConfigTrialGroup :: {self._experiment_id} cid:{self.tunable_config_id}" 

76 

77 def __eq__(self, other: Any) -> bool: 

78 if not isinstance(other, self.__class__): 

79 return False 

80 return ( 

81 self._tunable_config_id == other._tunable_config_id 

82 and self._experiment_id == other._experiment_id 

83 ) 

84 

85 @property 

86 @abstractmethod 

87 def tunable_config(self) -> TunableConfigData: 

88 """ 

89 Retrieve the (tunable) config data for this (tunable) config trial group from 

90 the storage. 

91 

92 Returns 

93 ------- 

94 TunableConfigData 

95 """ 

96 

97 @property 

98 @abstractmethod 

99 def trials(self) -> Dict[int, "TrialData"]: 

100 """ 

101 Retrieve the trials' data for this (tunable) config trial group from the 

102 storage. 

103 

104 Returns 

105 ------- 

106 trials : Dict[int, TrialData] 

107 A dictionary of the trials' data, keyed by trial id. 

108 """ 

109 

110 @property 

111 @abstractmethod 

112 def results_df(self) -> pandas.DataFrame: 

113 """ 

114 Retrieve all results for this (tunable) config trial group as a single 

115 DataFrame. 

116 

117 Returns 

118 ------- 

119 results : pandas.DataFrame 

120 A DataFrame with configurations and results from all trials of the experiment. 

121 Has columns [trial_id, config_id, ts_start, ts_end, status] 

122 followed by tunable config parameters (prefixed with "config.") and 

123 trial results (prefixed with "result."). The latter can be NULLs if the 

124 trial was not successful. 

125 

126 See Also 

127 -------- 

128 :py:attr:`mlos_bench.storage.base_experiment_data.ExperimentData.results_df` 

129 """