Coverage for mlos_bench/mlos_bench/tests/environments/local/local_env_stdout_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for extracting data from LocalEnv stdout. 

7""" 

8 

9import sys 

10 

11from mlos_bench.tunables.tunable_groups import TunableGroups 

12from mlos_bench.tests.environments import check_env_success 

13from mlos_bench.tests.environments.local import create_local_env 

14 

15 

16def test_local_env_stdout(tunable_groups: TunableGroups) -> None: 

17 """ 

18 Print benchmark results to stdout and capture them in the LocalEnv. 

19 """ 

20 local_env = create_local_env(tunable_groups, { 

21 "run": [ 

22 "echo 'Benchmark results:'", # This line should be ignored 

23 "echo 'latency,111'", 

24 "echo 'throughput,222'", 

25 "echo 'score,0.999'", 

26 "echo 'a,0,b,1'", 

27 ], 

28 "results_stdout_pattern": r"(\w+),([0-9.]+)", 

29 }) 

30 

31 check_env_success( 

32 local_env, tunable_groups, 

33 expected_results={ 

34 "latency": 111.0, 

35 "throughput": 222.0, 

36 "score": 0.999, 

37 "a": 0, 

38 "b": 1, 

39 }, 

40 expected_telemetry=[], 

41 ) 

42 

43 

44def test_local_env_stdout_anchored(tunable_groups: TunableGroups) -> None: 

45 """ 

46 Print benchmark results to stdout and capture them in the LocalEnv. 

47 """ 

48 local_env = create_local_env(tunable_groups, { 

49 "run": [ 

50 "echo 'Benchmark results:'", # This line should be ignored 

51 "echo 'latency,111'", 

52 "echo 'throughput,222'", 

53 "echo 'score,0.999'", 

54 "echo 'a,0,b,1'", # This line should be ignored in the case of anchored pattern 

55 ], 

56 "results_stdout_pattern": r"^(\w+),([0-9.]+)$", 

57 }) 

58 

59 check_env_success( 

60 local_env, tunable_groups, 

61 expected_results={ 

62 "latency": 111.0, 

63 "throughput": 222.0, 

64 "score": 0.999, 

65 # a, b are missing here 

66 }, 

67 expected_telemetry=[], 

68 ) 

69 

70 

71def test_local_env_file_stdout(tunable_groups: TunableGroups) -> None: 

72 """ 

73 Print benchmark results to *BOTH* stdout and a file and extract the results from both. 

74 """ 

75 local_env = create_local_env(tunable_groups, { 

76 "run": [ 

77 "echo 'latency,111'", 

78 "echo 'throughput,222'", 

79 "echo 'score,0.999'", 

80 "echo 'stdout-msg,string'", 

81 "echo '-------------------'", # Should be ignored 

82 "echo 'metric,value' > output.csv", 

83 "echo 'extra1,333' >> output.csv", 

84 "echo 'extra2,444' >> output.csv", 

85 "echo 'file-msg,string' >> output.csv", 

86 ], 

87 "results_stdout_pattern": r"([a-zA-Z0-9_-]+),([a-z0-9.]+)", 

88 "read_results_file": "output.csv", 

89 }) 

90 

91 check_env_success( 

92 local_env, tunable_groups, 

93 expected_results={ 

94 "latency": 111.0, 

95 "throughput": 222.0, 

96 "score": 0.999, 

97 "stdout-msg": "string", 

98 "extra1": 333.0, 

99 "extra2": 444.0, 

100 "file-msg": "string " if sys.platform == "win32" else "string", 

101 }, 

102 expected_telemetry=[], 

103 )