Coverage for mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Base interface for accessing the stored benchmark config trial group data. 

7""" 

8 

9from abc import ABCMeta, abstractmethod 

10from typing import Any, Dict, Optional, TYPE_CHECKING 

11 

12import pandas 

13 

14from mlos_bench.storage.base_tunable_config_data import TunableConfigData 

15 

16if TYPE_CHECKING: 

17 from mlos_bench.storage.base_trial_data import TrialData 

18 

19 

20class TunableConfigTrialGroupData(metaclass=ABCMeta): 

21 """ 

22 Base interface for accessing the stored experiment benchmark tunable config 

23 trial group data. 

24 

25 A (tunable) config is used to define an instance of values for a set of tunable 

26 parameters for a given experiment and can be used by one or more trial instances 

27 (e.g., for repeats), which we call a (tunable) config trial group. 

28 """ 

29 

30 def __init__(self, *, 

31 experiment_id: str, 

32 tunable_config_id: int, 

33 tunable_config_trial_group_id: Optional[int] = None): 

34 self._experiment_id = experiment_id 

35 self._tunable_config_id = tunable_config_id 

36 # can be lazily initialized as necessary: 

37 self._tunable_config_trial_group_id: Optional[int] = tunable_config_trial_group_id 

38 

39 @property 

40 def experiment_id(self) -> str: 

41 """ 

42 ID of the experiment. 

43 """ 

44 return self._experiment_id 

45 

46 @property 

47 def tunable_config_id(self) -> int: 

48 """ 

49 ID of the config. 

50 """ 

51 return self._tunable_config_id 

52 

53 @abstractmethod 

54 def _get_tunable_config_trial_group_id(self) -> int: 

55 """ 

56 Retrieve the trial's config_trial_group_id from the storage. 

57 """ 

58 raise NotImplementedError("subclass must implement") 

59 

60 @property 

61 def tunable_config_trial_group_id(self) -> int: 

62 """ 

63 The unique ID (within this experiment) of the (tunable) config trial group. 

64 

65 This is a unique identifier for all trials in this experiment using the given 

66 config_id, and typically defined as the the minimum trial_id for the given 

67 config_id. 

68 """ 

69 if self._tunable_config_trial_group_id is None: 

70 self._tunable_config_trial_group_id = self._get_tunable_config_trial_group_id() 

71 assert self._tunable_config_trial_group_id is not None 

72 return self._tunable_config_trial_group_id 

73 

74 def __repr__(self) -> str: 

75 return f"TunableConfigTrialGroup :: {self._experiment_id} cid:{self.tunable_config_id}" 

76 

77 def __eq__(self, other: Any) -> bool: 

78 if not isinstance(other, self.__class__): 

79 return False 

80 return self._tunable_config_id == other._tunable_config_id and self._experiment_id == other._experiment_id 

81 

82 @property 

83 @abstractmethod 

84 def tunable_config(self) -> TunableConfigData: 

85 """ 

86 Retrieve the (tunable) config data for this (tunable) config trial group from the storage. 

87 

88 Returns 

89 ------- 

90 TunableConfigData 

91 """ 

92 

93 @property 

94 @abstractmethod 

95 def trials(self) -> Dict[int, "TrialData"]: 

96 """ 

97 Retrieve the trials' data for this (tunable) config trial group from the storage. 

98 

99 Returns 

100 ------- 

101 trials : Dict[int, TrialData] 

102 A dictionary of the trials' data, keyed by trial id. 

103 """ 

104 

105 @property 

106 @abstractmethod 

107 def results_df(self) -> pandas.DataFrame: 

108 """ 

109 Retrieve all results for this (tunable) config trial group as a single DataFrame. 

110 

111 Returns 

112 ------- 

113 results : pandas.DataFrame 

114 A DataFrame with configurations and results from all trials of the experiment. 

115 Has columns [trial_id, config_id, ts_start, ts_end, status] 

116 followed by tunable config parameters (prefixed with "config.") and 

117 trial results (prefixed with "result."). The latter can be NULLs if the 

118 trial was not successful. 

119 

120 See Also 

121 -------- 

122 ExperimentData.results 

123 """