Coverage for mlos_bench/mlos_bench/tests/environments/local/composite_local_env_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for the composition of several LocalEnv benchmark environments.""" 

6import sys 

7from datetime import datetime, timedelta, tzinfo 

8 

9import pytest 

10from pytz import UTC 

11 

12from mlos_bench.tests import ZONE_INFO 

13from mlos_bench.tests.environments import check_env_success 

14from mlos_bench.tests.environments.local import create_composite_local_env 

15from mlos_bench.tunables.tunable_groups import TunableGroups 

16 

17 

18def _format_str(zone_info: tzinfo | None) -> str: 

19 if zone_info is not None: 

20 return "%Y-%m-%d %H:%M:%S.%f %z" 

21 return "%Y-%m-%d %H:%M:%S.%f" 

22 

23 

24# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

25@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

26def test_composite_env(tunable_groups: TunableGroups, zone_info: tzinfo | None) -> None: 

27 """ 

28 Produce benchmark and telemetry data in TWO local environments and combine the 

29 results. 

30 

31 Also checks that global configs flow down at least one level of CompositeEnv 

32 to its children without being explicitly specified in the CompositeEnv so they 

33 can be used in the shell_envs by its children. 

34 See Also: http://github.com/microsoft/MLOS/issues/501 

35 """ 

36 ts1 = datetime.now(zone_info) 

37 ts2 = ts1 + timedelta(minutes=2) 

38 

39 format_str = _format_str(zone_info) 

40 time_str1 = ts1.strftime(format_str) 

41 time_str2 = ts2.strftime(format_str) 

42 

43 (var_prefix, var_suffix) = ("%", "%") if sys.platform == "win32" else ("$", "") 

44 

45 env = create_composite_local_env( 

46 tunable_groups=tunable_groups, 

47 global_config={ 

48 "reads": 2222, 

49 "writes": 1111, 

50 }, 

51 params={ 

52 "const_args": { 

53 "latency": 4.2, 

54 "throughput": 768, 

55 "errors": 0, 

56 } 

57 }, 

58 local_configs=[ 

59 { 

60 "const_args": { 

61 "latency": 3.3, 

62 "reads": 0, 

63 }, 

64 "required_args": ["errors", "reads"], 

65 "shell_env_params": [ 

66 # const_args overridden by the composite env 

67 "latency", 

68 # Comes from the parent const_args 

69 "errors", 

70 # const_args overridden by the global config 

71 "reads", 

72 ], 

73 "run": [ 

74 "echo 'metric,value' > output.csv", 

75 f"echo 'latency,{var_prefix}latency{var_suffix}' >> output.csv", 

76 f"echo 'errors,{var_prefix}errors{var_suffix}' >> output.csv", 

77 f"echo 'reads,{var_prefix}reads{var_suffix}' >> output.csv", 

78 "echo '-------------------'", # This output does not go anywhere 

79 "echo 'timestamp,metric,value' > telemetry.csv", 

80 f"echo {time_str1},cpu_load,0.64 >> telemetry.csv", 

81 f"echo {time_str1},mem_usage,5120 >> telemetry.csv", 

82 ], 

83 "read_results_file": "output.csv", 

84 "read_telemetry_file": "telemetry.csv", 

85 }, 

86 { 

87 "const_args": { 

88 "throughput": 999, 

89 "score": 0.97, 

90 }, 

91 "required_args": ["writes"], 

92 "shell_env_params": [ 

93 # const_args overridden by the composite env 

94 "throughput", 

95 # Comes from the local const_args 

96 "score", 

97 # Comes straight from the global config 

98 "writes", 

99 ], 

100 "run": [ 

101 "echo 'metric,value' > output.csv", 

102 f"echo 'throughput,{var_prefix}throughput{var_suffix}' >> output.csv", 

103 f"echo 'score,{var_prefix}score{var_suffix}' >> output.csv", 

104 f"echo 'writes,{var_prefix}writes{var_suffix}' >> output.csv", 

105 "echo '-------------------'", # This output does not go anywhere 

106 "echo 'timestamp,metric,value' > telemetry.csv", 

107 f"echo {time_str2},cpu_load,0.79 >> telemetry.csv", 

108 f"echo {time_str2},mem_usage,40960 >> telemetry.csv", 

109 ], 

110 "read_results_file": "output.csv", 

111 "read_telemetry_file": "telemetry.csv", 

112 }, 

113 ], 

114 ) 

115 

116 check_env_success( 

117 env, 

118 tunable_groups, 

119 expected_results={ 

120 "latency": 4.2, 

121 "throughput": 768.0, 

122 "score": 0.97, 

123 "errors": 0.0, 

124 "reads": 2222.0, 

125 "writes": 1111.0, 

126 }, 

127 expected_telemetry=[ 

128 (ts1.astimezone(UTC), "cpu_load", 0.64), 

129 (ts1.astimezone(UTC), "mem_usage", 5120.0), 

130 (ts2.astimezone(UTC), "cpu_load", 0.79), 

131 (ts2.astimezone(UTC), "mem_usage", 40960.0), 

132 ], 

133 )