Coverage for mlos_bench/mlos_bench/config/environments/apps/fio/scripts/local/process_fio_results.py: 41%

1#!/usr/bin/env python3 

2# 

3# Copyright (c) Microsoft Corporation. 

4# Licensed under the MIT License. 

5# 

6"""Script for post-processing FIO results for mlos_bench.""" 

7 

8import argparse 

9import itertools 

10import json 

11from typing import Any, Iterator, Tuple 

12 

13import pandas 

14 

15 

16def _flat_dict(data: Any, path: str) -> Iterator[Tuple[str, Any]]: 

17 """Flatten every dict in the hierarchy and rename the keys with the dict path.""" 

18 if isinstance(data, dict): 

19 for key, val in data.items(): 

20 yield from _flat_dict(val, f"{path}.{key}") 

21 else: 

22 yield (path, data) 

23 

24 

25def _main(input_file: str, output_file: str, prefix: str) -> None: 

26 """Convert FIO read data from JSON to tall CSV.""" 

27 with open(input_file, mode="r", encoding="utf-8") as fh_input: 

28 json_data = json.load(fh_input) 

29 

30 data = list( 

31 itertools.chain( 

32 _flat_dict(json_data["jobs"][0], prefix), 

33 _flat_dict(json_data["disk_util"][0], f"{prefix}.disk_util"), 

34 ) 

35 ) 

36 

37 tall_df = pandas.DataFrame(data, columns=["metric", "value"]) 

38 tall_df.to_csv(output_file, index=False) 

39 print(f"Converted: {input_file} -> {output_file}") 

40 # print(tall_df) 

41 

42 

43if __name__ == "__main__": 

44 

45 parser = argparse.ArgumentParser(description="Post-process FIO benchmark results.") 

46 

47 parser.add_argument( 

48 "input", 

49 help="FIO benchmark results in JSON format (downloaded from a remote VM).", 

50 ) 

51 parser.add_argument( 

52 "output", 

53 help="Converted FIO benchmark data (CSV, to be consumed by mlos_bench).", 

54 ) 

55 parser.add_argument( 

56 "--prefix", 

57 default="fio", 

58 help="Prefix of the metric IDs (default 'fio')", 

59 ) 

60 

61 args = parser.parse_args() 

62 _main(args.input, args.output, args.prefix)