Coverage for mlos_bench/mlos_bench/tests/config/schemas/storage/test_storage_schemas.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Tests for storage schema validation. 

7""" 

8 

9from os import path 

10 

11import pytest 

12 

13from mlos_core.tests import get_all_concrete_subclasses 

14 

15from mlos_bench.config.schemas import ConfigSchema 

16from mlos_bench.storage.base_storage import Storage 

17 

18from mlos_bench.tests import try_resolve_class_name 

19from mlos_bench.tests.config.schemas import (get_schema_test_cases, 

20 check_test_case_against_schema, 

21 check_test_case_config_with_extra_param) 

22 

23 

24# General testing strategy: 

25# - hand code a set of good/bad configs (useful to test editor schema checking) 

26# - for each config, load and validate against expected schema 

27 

28TEST_CASES = get_schema_test_cases(path.join(path.dirname(__file__), "test-cases")) 

29 

30# Dynamically enumerate some of the cases we want to make sure we cover. 

31 

32expected_mlos_bench_storage_class_names = [subclass.__module__ + "." + subclass.__name__ 

33 for subclass in get_all_concrete_subclasses(Storage, # type: ignore[type-abstract] 

34 pkg_name='mlos_bench')] 

35assert expected_mlos_bench_storage_class_names 

36 

37# Do the full cross product of all the test cases and all the storage types. 

38 

39 

40@pytest.mark.parametrize("test_case_subtype", sorted(TEST_CASES.by_subtype)) 

41@pytest.mark.parametrize("mlos_bench_storage_type", expected_mlos_bench_storage_class_names) 

42def test_case_coverage_mlos_bench_storage_type(test_case_subtype: str, mlos_bench_storage_type: str) -> None: 

43 """ 

44 Checks to see if there is a given type of test case for the given mlos_bench storage type. 

45 """ 

46 for test_case in TEST_CASES.by_subtype[test_case_subtype].values(): 

47 if try_resolve_class_name(test_case.config.get("class")) == mlos_bench_storage_type: 

48 return 

49 raise NotImplementedError( 

50 f"Missing test case for subtype {test_case_subtype} for Storage class {mlos_bench_storage_type}") 

51 

52 

53# Now we actually perform all of those validation tests. 

54 

55@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_path)) 

56def test_storage_configs_against_schema(test_case_name: str) -> None: 

57 """ 

58 Checks that the storage config validates against the schema. 

59 """ 

60 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.STORAGE) 

61 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED) 

62 

63 

64@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_type["good"])) 

65def test_storage_configs_with_extra_param(test_case_name: str) -> None: 

66 """ 

67 Checks that the storage config fails to validate if extra params are present in certain places. 

68 """ 

69 check_test_case_config_with_extra_param(TEST_CASES.by_type["good"][test_case_name], ConfigSchema.STORAGE) 

70 check_test_case_config_with_extra_param(TEST_CASES.by_type["good"][test_case_name], ConfigSchema.UNIFIED) 

71 

72 

73if __name__ == '__main__': 

74 pytest.main([__file__, '-n0'],)