Coverage for mlos_bench/mlos_bench/tests/config/schemas/cli/test_cli_schemas.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Tests for CLI schema validation. 

7""" 

8 

9from os import path 

10 

11import pytest 

12 

13from mlos_bench.config.schemas import ConfigSchema 

14 

15from mlos_bench.tests.config.schemas import (get_schema_test_cases, 

16 check_test_case_against_schema, 

17 check_test_case_config_with_extra_param) 

18 

19 

20# General testing strategy: 

21# - hand code a set of good/bad configs (useful to test editor schema checking) 

22# - for each config, load and validate against expected schema 

23 

24TEST_CASES = get_schema_test_cases(path.join(path.dirname(__file__), "test-cases")) 

25 

26 

27# Now we actually perform all of those validation tests. 

28 

29@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_path)) 

30def test_cli_configs_against_schema(test_case_name: str) -> None: 

31 """ 

32 Checks that the CLI config validates against the schema. 

33 """ 

34 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.CLI) 

35 if TEST_CASES.by_path[test_case_name].test_case_type != "bad": 

36 # Unified schema has a hard time validating bad configs, so we skip it. 

37 # The trouble is that tunable-values, cli, globals all look like flat dicts with minor constraints on them, 

38 # so adding/removing params doesn't invalidate it against all of the config types. 

39 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED) 

40 

41 

42@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_type["good"])) 

43def test_cli_configs_with_extra_param(test_case_name: str) -> None: 

44 """ 

45 Checks that the cli config fails to validate if extra params are present in certain places. 

46 """ 

47 check_test_case_config_with_extra_param(TEST_CASES.by_type["good"][test_case_name], ConfigSchema.CLI) 

48 if TEST_CASES.by_path[test_case_name].test_case_type != "bad": 

49 # Unified schema has a hard time validating bad configs, so we skip it. 

50 # The trouble is that tunable-values, cli, globals all look like flat dicts with minor constraints on them, 

51 # so adding/removing params doesn't invalidate it against all of the config types. 

52 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED)