Coverage for mlos_bench/mlos_bench/tests/optimizers/llamatune_opt_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5""" 

6Unit tests for mock mlos_bench optimizer. 

7""" 

8 

9import pytest 

10 

11from mlos_bench.environments.status import Status 

12from mlos_bench.tunables.tunable_groups import TunableGroups 

13from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

14 

15from mlos_bench.tests import SEED 

16 

17# pylint: disable=redefined-outer-name 

18 

19 

20@pytest.fixture 

21def llamatune_opt(tunable_groups: TunableGroups) -> MlosCoreOptimizer: 

22 """ 

23 Test fixture for mlos_core SMAC optimizer. 

24 """ 

25 return MlosCoreOptimizer( 

26 tunables=tunable_groups, 

27 service=None, 

28 config={ 

29 "space_adapter_type": "LLAMATUNE", 

30 "space_adapter_config": { 

31 "num_low_dims": 2, 

32 }, 

33 "optimization_targets": {"score": "min"}, 

34 "max_suggestions": 10, 

35 "optimizer_type": "SMAC", 

36 "seed": SEED, 

37 # "start_with_defaults": False, 

38 }) 

39 

40 

41@pytest.fixture 

42def mock_scores() -> list: 

43 """ 

44 A list of fake benchmark scores to test the optimizers. 

45 """ 

46 return [88.88, 66.66, 99.99] 

47 

48 

49def test_llamatune_optimizer(llamatune_opt: MlosCoreOptimizer, mock_scores: list) -> None: 

50 """ 

51 Make sure that llamatune+smac optimizer initializes and works correctly. 

52 """ 

53 for score in mock_scores: 

54 assert llamatune_opt.not_converged() 

55 tunables = llamatune_opt.suggest() 

56 # FIXME: Emukit optimizer is not deterministic, so we can't check the tunables here. 

57 llamatune_opt.register(tunables, Status.SUCCEEDED, score) 

58 

59 (score, _tunables) = llamatune_opt.get_best_observation() 

60 assert score == pytest.approx(66.66, 0.01) 

61 

62 

63if __name__ == '__main__': 

64 # For attaching debugger debugging: 

65 pytest.main(["-vv", "-n1", "-k", "test_llamatune_optimizer", __file__])