Coverage for mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Base interface for accessing the stored benchmark config trial group data. 

7 

8Since a single config may be used by multiple trials, we can group them together for 

9easier analysis. 

10 

11See Also 

12-------- 

13:py:mod:`mlos_bench.storage` : 

14 The base storage module for mlos_bench, which includes some basic examples in 

15 the documentation. 

16""" 

17 

18from abc import ABCMeta, abstractmethod 

19from typing import TYPE_CHECKING, Any 

20 

21import pandas 

22 

23from mlos_bench.storage.base_tunable_config_data import TunableConfigData 

24 

25if TYPE_CHECKING: 

26 from mlos_bench.storage.base_trial_data import TrialData 

27 

28 

29class TunableConfigTrialGroupData(metaclass=ABCMeta): 

30 """ 

31 Base interface for accessing the stored experiment benchmark tunable config trial 

32 group data. 

33 

34 A (tunable) config is used to define an instance of values for a set of tunable 

35 parameters for a given experiment and can be used by one or more trial instances 

36 (e.g., for repeats), which we call a (tunable) config trial group. 

37 """ 

38 

39 def __init__( 

40 self, 

41 *, 

42 experiment_id: str, 

43 tunable_config_id: int, 

44 tunable_config_trial_group_id: int | None = None, 

45 ): 

46 self._experiment_id = experiment_id 

47 self._tunable_config_id = tunable_config_id 

48 # can be lazily initialized as necessary: 

49 self._tunable_config_trial_group_id: int | None = tunable_config_trial_group_id 

50 

51 @property 

52 def experiment_id(self) -> str: 

53 """ID of the experiment.""" 

54 return self._experiment_id 

55 

56 @property 

57 def tunable_config_id(self) -> int: 

58 """ID of the config.""" 

59 return self._tunable_config_id 

60 

61 @abstractmethod 

62 def _get_tunable_config_trial_group_id(self) -> int: 

63 """Retrieve the trial's config_trial_group_id from the storage.""" 

64 raise NotImplementedError("subclass must implement") 

65 

66 @property 

67 def tunable_config_trial_group_id(self) -> int: 

68 """ 

69 The unique ID (within this experiment) of the (tunable) config trial group. 

70 

71 This is a unique identifier for all trials in this experiment using the given 

72 config_id, and typically defined as the the minimum trial_id for the given 

73 config_id. 

74 """ 

75 if self._tunable_config_trial_group_id is None: 

76 self._tunable_config_trial_group_id = self._get_tunable_config_trial_group_id() 

77 assert self._tunable_config_trial_group_id is not None 

78 return self._tunable_config_trial_group_id 

79 

80 def __repr__(self) -> str: 

81 return f"TunableConfigTrialGroup :: {self._experiment_id} cid:{self.tunable_config_id}" 

82 

83 def __eq__(self, other: Any) -> bool: 

84 if not isinstance(other, self.__class__): 

85 return False 

86 return ( 

87 self._tunable_config_id == other._tunable_config_id 

88 and self._experiment_id == other._experiment_id 

89 ) 

90 

91 @property 

92 @abstractmethod 

93 def tunable_config(self) -> TunableConfigData: 

94 """ 

95 Retrieve the (tunable) config data for this (tunable) config trial group from 

96 the storage. 

97 

98 Returns 

99 ------- 

100 TunableConfigData 

101 """ 

102 

103 @property 

104 @abstractmethod 

105 def trials(self) -> dict[int, "TrialData"]: 

106 """ 

107 Retrieve the trials' data for this (tunable) config trial group from the 

108 storage. 

109 

110 Returns 

111 ------- 

112 trials : dict[int, TrialData] 

113 A dictionary of the trials' data, keyed by trial id. 

114 """ 

115 

116 @property 

117 @abstractmethod 

118 def results_df(self) -> pandas.DataFrame: 

119 """ 

120 Retrieve all results for this (tunable) config trial group as a single 

121 DataFrame. 

122 

123 Returns 

124 ------- 

125 results : pandas.DataFrame 

126 A DataFrame with configurations and results from all trials of the experiment. 

127 Has columns [trial_id, config_id, ts_start, ts_end, status] 

128 followed by tunable config parameters (prefixed with "config.") and 

129 trial results (prefixed with "result."). The latter can be NULLs if the 

130 trial was not successful. 

131 

132 See Also 

133 -------- 

134 :py:attr:`mlos_bench.storage.base_experiment_data.ExperimentData.results_df` 

135 """