Coverage for mlos_bench/mlos_bench/tests/environments/local/composite_local_env_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for the composition of several LocalEnv benchmark environments. 

7""" 

8import sys 

9from datetime import datetime, timedelta, tzinfo 

10from typing import Optional 

11 

12from pytz import UTC 

13import pytest 

14 

15from mlos_bench.tunables.tunable_groups import TunableGroups 

16from mlos_bench.tests.environments import check_env_success 

17from mlos_bench.tests.environments.local import create_composite_local_env 

18from mlos_bench.tests import ZONE_INFO 

19 

20 

21def _format_str(zone_info: Optional[tzinfo]) -> str: 

22 if zone_info is not None: 

23 return "%Y-%m-%d %H:%M:%S %z" 

24 return "%Y-%m-%d %H:%M:%S" 

25 

26 

27# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

28@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

29def test_composite_env(tunable_groups: TunableGroups, zone_info: Optional[tzinfo]) -> None: 

30 """ 

31 Produce benchmark and telemetry data in TWO local environments 

32 and combine the results. 

33 Also checks that global configs flow down at least one level of CompositeEnv 

34 to its children without being explicitly specified in the CompositeEnv so they 

35 can be used in the shell_envs by its children. 

36 See Also: http://github.com/microsoft/MLOS/issues/501 

37 """ 

38 ts1 = datetime.now(zone_info) 

39 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

40 ts2 = ts1 + timedelta(minutes=2) 

41 

42 format_str = _format_str(zone_info) 

43 time_str1 = ts1.strftime(format_str) 

44 time_str2 = ts2.strftime(format_str) 

45 

46 (var_prefix, var_suffix) = ("%", "%") if sys.platform == 'win32' else ("$", "") 

47 

48 env = create_composite_local_env( 

49 tunable_groups=tunable_groups, 

50 global_config={ 

51 "reads": 2222, 

52 "writes": 1111, 

53 }, 

54 params={ 

55 "const_args": { 

56 "latency": 4.2, 

57 "throughput": 768, 

58 "errors": 0, 

59 } 

60 }, 

61 local_configs=[ 

62 { 

63 "const_args": { 

64 "latency": 3.3, 

65 "reads": 0, 

66 }, 

67 "required_args": ["errors", "reads"], 

68 "shell_env_params": [ 

69 "latency", # const_args overridden by the composite env 

70 "errors", # Comes from the parent const_args 

71 "reads" # const_args overridden by the global config 

72 ], 

73 "run": [ 

74 "echo 'metric,value' > output.csv", 

75 f"echo 'latency,{var_prefix}latency{var_suffix}' >> output.csv", 

76 f"echo 'errors,{var_prefix}errors{var_suffix}' >> output.csv", 

77 f"echo 'reads,{var_prefix}reads{var_suffix}' >> output.csv", 

78 "echo '-------------------'", # This output does not go anywhere 

79 "echo 'timestamp,metric,value' > telemetry.csv", 

80 f"echo {time_str1},cpu_load,0.64 >> telemetry.csv", 

81 f"echo {time_str1},mem_usage,5120 >> telemetry.csv", 

82 ], 

83 "read_results_file": "output.csv", 

84 "read_telemetry_file": "telemetry.csv", 

85 }, 

86 { 

87 "const_args": { 

88 "throughput": 999, 

89 "score": 0.97, 

90 }, 

91 "required_args": ["writes"], 

92 "shell_env_params": [ 

93 "throughput", # const_args overridden by the composite env 

94 "score", # Comes from the local const_args 

95 "writes" # Comes straight from the global config 

96 ], 

97 "run": [ 

98 "echo 'metric,value' > output.csv", 

99 f"echo 'throughput,{var_prefix}throughput{var_suffix}' >> output.csv", 

100 f"echo 'score,{var_prefix}score{var_suffix}' >> output.csv", 

101 f"echo 'writes,{var_prefix}writes{var_suffix}' >> output.csv", 

102 "echo '-------------------'", # This output does not go anywhere 

103 "echo 'timestamp,metric,value' > telemetry.csv", 

104 f"echo {time_str2},cpu_load,0.79 >> telemetry.csv", 

105 f"echo {time_str2},mem_usage,40960 >> telemetry.csv", 

106 ], 

107 "read_results_file": "output.csv", 

108 "read_telemetry_file": "telemetry.csv", 

109 } 

110 ] 

111 ) 

112 

113 check_env_success( 

114 env, tunable_groups, 

115 expected_results={ 

116 "latency": 4.2, 

117 "throughput": 768.0, 

118 "score": 0.97, 

119 "errors": 0.0, 

120 "reads": 2222.0, 

121 "writes": 1111.0, 

122 }, 

123 expected_telemetry=[ 

124 (ts1.astimezone(UTC), "cpu_load", 0.64), 

125 (ts1.astimezone(UTC), "mem_usage", 5120.0), 

126 (ts2.astimezone(UTC), "cpu_load", 0.79), 

127 (ts2.astimezone(UTC), "mem_usage", 40960.0), 

128 ], 

129 )