Coverage for mlos_bench/mlos_bench/tests/environments/local/local_env_telemetry_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for telemetry and status of LocalEnv benchmark environment. 

7""" 

8from datetime import datetime, timedelta, tzinfo 

9from typing import Optional 

10 

11from pytz import UTC 

12 

13import pytest 

14 

15from mlos_bench.tunables.tunable_groups import TunableGroups 

16from mlos_bench.tests import ZONE_INFO 

17from mlos_bench.tests.environments import check_env_success, check_env_fail_telemetry 

18from mlos_bench.tests.environments.local import create_local_env 

19 

20 

21def _format_str(zone_info: Optional[tzinfo]) -> str: 

22 if zone_info is not None: 

23 return "%Y-%m-%d %H:%M:%S %z" 

24 return "%Y-%m-%d %H:%M:%S" 

25 

26 

27# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

28@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

29def test_local_env_telemetry(tunable_groups: TunableGroups, zone_info: Optional[tzinfo]) -> None: 

30 """ 

31 Produce benchmark and telemetry data in a local script and read it. 

32 """ 

33 ts1 = datetime.now(zone_info) 

34 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

35 ts2 = ts1 + timedelta(minutes=1) 

36 

37 format_str = _format_str(zone_info) 

38 time_str1 = ts1.strftime(format_str) 

39 time_str2 = ts2.strftime(format_str) 

40 

41 local_env = create_local_env(tunable_groups, { 

42 "run": [ 

43 "echo 'metric,value' > output.csv", 

44 "echo 'latency,4.1' >> output.csv", 

45 "echo 'throughput,512' >> output.csv", 

46 "echo 'score,0.95' >> output.csv", 

47 "echo '-------------------'", # This output does not go anywhere 

48 "echo 'timestamp,metric,value' > telemetry.csv", 

49 f"echo {time_str1},cpu_load,0.65 >> telemetry.csv", 

50 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

51 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

52 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

53 ], 

54 "read_results_file": "output.csv", 

55 "read_telemetry_file": "telemetry.csv", 

56 }) 

57 

58 check_env_success( 

59 local_env, tunable_groups, 

60 expected_results={ 

61 "latency": 4.1, 

62 "throughput": 512.0, 

63 "score": 0.95, 

64 }, 

65 expected_telemetry=[ 

66 (ts1.astimezone(UTC), "cpu_load", 0.65), 

67 (ts1.astimezone(UTC), "mem_usage", 10240.0), 

68 (ts2.astimezone(UTC), "cpu_load", 0.8), 

69 (ts2.astimezone(UTC), "mem_usage", 20480.0), 

70 ], 

71 ) 

72 

73 

74# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

75@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

76def test_local_env_telemetry_no_header(tunable_groups: TunableGroups, zone_info: Optional[tzinfo]) -> None: 

77 """ 

78 Read the telemetry data with no header. 

79 """ 

80 ts1 = datetime.now(zone_info) 

81 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

82 ts2 = ts1 + timedelta(minutes=1) 

83 

84 format_str = _format_str(zone_info) 

85 time_str1 = ts1.strftime(format_str) 

86 time_str2 = ts2.strftime(format_str) 

87 

88 local_env = create_local_env(tunable_groups, { 

89 "run": [ 

90 f"echo {time_str1},cpu_load,0.65 > telemetry.csv", 

91 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

92 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

93 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

94 ], 

95 "read_telemetry_file": "telemetry.csv", 

96 }) 

97 

98 check_env_success( 

99 local_env, tunable_groups, 

100 expected_results={}, 

101 expected_telemetry=[ 

102 (ts1.astimezone(UTC), "cpu_load", 0.65), 

103 (ts1.astimezone(UTC), "mem_usage", 10240.0), 

104 (ts2.astimezone(UTC), "cpu_load", 0.8), 

105 (ts2.astimezone(UTC), "mem_usage", 20480.0), 

106 ], 

107 ) 

108 

109 

110@pytest.mark.filterwarnings("ignore:.*(Could not infer format, so each element will be parsed individually, falling back to `dateutil`).*:UserWarning::0") # pylint: disable=line-too-long # noqa 

111@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

112def test_local_env_telemetry_wrong_header(tunable_groups: TunableGroups, zone_info: Optional[tzinfo]) -> None: 

113 """ 

114 Read the telemetry data with incorrect header. 

115 """ 

116 ts1 = datetime.now(zone_info) 

117 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

118 ts2 = ts1 + timedelta(minutes=1) 

119 

120 format_str = _format_str(zone_info) 

121 time_str1 = ts1.strftime(format_str) 

122 time_str2 = ts2.strftime(format_str) 

123 

124 local_env = create_local_env(tunable_groups, { 

125 "run": [ 

126 # Error: the data is correct, but the header has unexpected column names 

127 "echo 'ts,metric_name,metric_value' > telemetry.csv", 

128 f"echo {time_str1},cpu_load,0.65 >> telemetry.csv", 

129 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

130 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

131 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

132 ], 

133 "read_telemetry_file": "telemetry.csv", 

134 }) 

135 

136 check_env_fail_telemetry(local_env, tunable_groups) 

137 

138 

139def test_local_env_telemetry_invalid(tunable_groups: TunableGroups) -> None: 

140 """ 

141 Fail when the telemetry data has wrong format. 

142 """ 

143 zone_info = UTC 

144 ts1 = datetime.now(zone_info) 

145 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

146 ts2 = ts1 + timedelta(minutes=1) 

147 

148 format_str = _format_str(zone_info) 

149 time_str1 = ts1.strftime(format_str) 

150 time_str2 = ts2.strftime(format_str) 

151 

152 local_env = create_local_env(tunable_groups, { 

153 "run": [ 

154 # Error: too many columns 

155 f"echo {time_str1},EXTRA,cpu_load,0.65 > telemetry.csv", 

156 f"echo {time_str1},EXTRA,mem_usage,10240 >> telemetry.csv", 

157 f"echo {time_str2},EXTRA,cpu_load,0.8 >> telemetry.csv", 

158 f"echo {time_str2},EXTRA,mem_usage,20480 >> telemetry.csv", 

159 ], 

160 "read_telemetry_file": "telemetry.csv", 

161 }) 

162 

163 check_env_fail_telemetry(local_env, tunable_groups) 

164 

165 

166def test_local_env_telemetry_invalid_ts(tunable_groups: TunableGroups) -> None: 

167 """ 

168 Fail when the telemetry data has wrong format. 

169 """ 

170 local_env = create_local_env(tunable_groups, { 

171 "run": [ 

172 # Error: field 1 must be a timestamp 

173 "echo 1,cpu_load,0.65 > telemetry.csv", 

174 "echo 2,mem_usage,10240 >> telemetry.csv", 

175 "echo 3,cpu_load,0.8 >> telemetry.csv", 

176 "echo 4,mem_usage,20480 >> telemetry.csv", 

177 ], 

178 "read_telemetry_file": "telemetry.csv", 

179 }) 

180 

181 check_env_fail_telemetry(local_env, tunable_groups)