Coverage for mlos_bench/mlos_bench/tests/config/schemas/optimizers/test_optimizer_schemas.py: 100%
54 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-06 00:35 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""
6Tests for optimizer schema validation.
7"""
9from os import path
10from typing import Optional
12import pytest
14from mlos_core.optimizers import OptimizerType
15from mlos_core.spaces.adapters import SpaceAdapterType
16from mlos_core.tests import get_all_concrete_subclasses
18from mlos_bench.config.schemas import ConfigSchema
19from mlos_bench.optimizers.base_optimizer import Optimizer
21from mlos_bench.tests import try_resolve_class_name
22from mlos_bench.tests.config.schemas import (get_schema_test_cases,
23 check_test_case_against_schema,
24 check_test_case_config_with_extra_param)
27# General testing strategy:
28# - hand code a set of good/bad configs (useful to test editor schema checking)
29# - enumerate and try to check that we've covered all the cases
30# - for each config, load and validate against expected schema
32TEST_CASES = get_schema_test_cases(path.join(path.dirname(__file__), "test-cases"))
35# Dynamically enumerate some of the cases we want to make sure we cover.
37expected_mlos_bench_optimizer_class_names = [subclass.__module__ + "." + subclass.__name__
38 for subclass in get_all_concrete_subclasses(Optimizer, # type: ignore[type-abstract]
39 pkg_name='mlos_bench')]
40assert expected_mlos_bench_optimizer_class_names
42# Also make sure that we check for configs where the optimizer_type or space_adapter_type are left unspecified (None).
44expected_mlos_core_optimizer_types = list(OptimizerType) + [None]
45assert expected_mlos_core_optimizer_types
47expected_mlos_core_space_adapter_types = list(SpaceAdapterType) + [None]
48assert expected_mlos_core_space_adapter_types
51# Do the full cross product of all the test cases and all the optimizer types.
52@pytest.mark.parametrize("test_case_subtype", sorted(TEST_CASES.by_subtype))
53@pytest.mark.parametrize("mlos_bench_optimizer_type", expected_mlos_bench_optimizer_class_names)
54def test_case_coverage_mlos_bench_optimizer_type(test_case_subtype: str, mlos_bench_optimizer_type: str) -> None:
55 """
56 Checks to see if there is a given type of test case for the given mlos_bench optimizer type.
57 """
58 for test_case in TEST_CASES.by_subtype[test_case_subtype].values():
59 if try_resolve_class_name(test_case.config.get("class")) == mlos_bench_optimizer_type:
60 return
61 raise NotImplementedError(
62 f"Missing test case for subtype {test_case_subtype} for Optimizer class {mlos_bench_optimizer_type}")
64# Being a little lazy for the moment and relaxing the requirement that we have
65# a subtype test case for each optimizer and space adapter combo.
68@pytest.mark.parametrize("test_case_type", sorted(TEST_CASES.by_type))
69# @pytest.mark.parametrize("test_case_subtype", sorted(TEST_CASES.by_subtype))
70@pytest.mark.parametrize("mlos_core_optimizer_type", expected_mlos_core_optimizer_types)
71def test_case_coverage_mlos_core_optimizer_type(test_case_type: str,
72 mlos_core_optimizer_type: Optional[OptimizerType]) -> None:
73 """
74 Checks to see if there is a given type of test case for the given mlos_core optimizer type.
75 """
76 optimizer_name = None if mlos_core_optimizer_type is None else mlos_core_optimizer_type.name
77 for test_case in TEST_CASES.by_type[test_case_type].values():
78 if try_resolve_class_name(test_case.config.get("class")) \
79 == "mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer":
80 optimizer_type = None
81 if test_case.config.get("config"):
82 optimizer_type = test_case.config["config"].get("optimizer_type", None)
83 if optimizer_type == optimizer_name:
84 return
85 raise NotImplementedError(
86 f"Missing test case for type {test_case_type} for MlosCore Optimizer type {mlos_core_optimizer_type}")
89@pytest.mark.parametrize("test_case_type", sorted(TEST_CASES.by_type))
90# @pytest.mark.parametrize("test_case_subtype", sorted(TEST_CASES.by_subtype))
91@pytest.mark.parametrize("mlos_core_space_adapter_type", expected_mlos_core_space_adapter_types)
92def test_case_coverage_mlos_core_space_adapter_type(test_case_type: str,
93 mlos_core_space_adapter_type: Optional[SpaceAdapterType]) -> None:
94 """
95 Checks to see if there is a given type of test case for the given mlos_core space adapter type.
96 """
97 space_adapter_name = None if mlos_core_space_adapter_type is None else mlos_core_space_adapter_type.name
98 for test_case in TEST_CASES.by_type[test_case_type].values():
99 if try_resolve_class_name(test_case.config.get("class")) \
100 == "mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer":
101 space_adapter_type = None
102 if test_case.config.get("config"):
103 space_adapter_type = test_case.config["config"].get("space_adapter_type", None)
104 if space_adapter_type == space_adapter_name:
105 return
106 raise NotImplementedError(
107 f"Missing test case for type {test_case_type} for SpaceAdapter type {mlos_core_space_adapter_type}")
110# Now we actually perform all of those validation tests.
112@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_path))
113def test_optimizer_configs_against_schema(test_case_name: str) -> None:
114 """
115 Checks that the optimizer config validates against the schema.
116 """
117 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.OPTIMIZER)
118 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED)
121@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_type["good"]))
122def test_optimizer_configs_with_extra_param(test_case_name: str) -> None:
123 """
124 Checks that the optimizer config fails to validate if extra params are present in certain places.
125 """
126 check_test_case_config_with_extra_param(TEST_CASES.by_type["good"][test_case_name], ConfigSchema.OPTIMIZER)
127 check_test_case_config_with_extra_param(TEST_CASES.by_type["good"][test_case_name], ConfigSchema.UNIFIED)