Coverage for mlos_bench/mlos_bench/tests/config/schemas/cli/test_cli_schemas.py: 100%
15 statements
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-20 00:44 +0000
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-20 00:44 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Tests for CLI schema validation."""
7from os import path
9import pytest
11from mlos_bench.config.schemas import ConfigSchema
12from mlos_bench.tests.config.schemas import (
13 check_test_case_against_schema,
14 check_test_case_config_with_extra_param,
15 get_schema_test_cases,
16)
18# General testing strategy:
19# - hand code a set of good/bad configs (useful to test editor schema checking)
20# - for each config, load and validate against expected schema
22TEST_CASES = get_schema_test_cases(path.join(path.dirname(__file__), "test-cases"))
25# Now we actually perform all of those validation tests.
28@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_path))
29def test_cli_configs_against_schema(test_case_name: str) -> None:
30 """Checks that the CLI config validates against the schema."""
31 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.CLI)
32 if TEST_CASES.by_path[test_case_name].test_case_type != "bad":
33 # Unified schema has a hard time validating bad configs, so we skip it.
34 # The trouble is that tunable-values, cli, globals all look like flat dicts
35 # with minor constraints on them, so adding/removing params doesn't
36 # invalidate it against all of the config types.
37 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED)
40@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_type["good"]))
41def test_cli_configs_with_extra_param(test_case_name: str) -> None:
42 """Checks that the cli config fails to validate if extra params are present in
43 certain places.
44 """
45 check_test_case_config_with_extra_param(
46 TEST_CASES.by_type["good"][test_case_name],
47 ConfigSchema.CLI,
48 )
49 if TEST_CASES.by_path[test_case_name].test_case_type != "bad":
50 # Unified schema has a hard time validating bad configs, so we skip it.
51 # The trouble is that tunable-values, cli, globals all look like flat dicts
52 # with minor constraints on them, so adding/removing params doesn't
53 # invalidate it against all of the config types.
54 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED)