UIA Control Detection

UIA control detection is a method to detect standard controls in the application using the UI Automation (UIA) framework. It provides a set of APIs to access and manipulate the UI elements in Windows applications.

Note

The UIA control detection may fail to detect non-standard controls or custom controls in the application.

Configuration

To activate the icon control filtering, you need to set CONTROL_BACKEND to ["uia"] in the config_dev.yaml file.

CONTROL_BACKEND: ["uia"]

Reference

The singleton facade class for control inspector.

Initialize the control inspector.

Parameters:
  • backend (str, default: 'uia' ) –

    The backend to use.

Source code in automator/ui_control/inspector.py
478
479
480
481
482
483
def __init__(self, backend: str = "uia") -> None:
    """
    Initialize the control inspector.
    :param backend: The backend to use.
    """
    self.backend = backend

desktop property

Get all the desktop windows.

Returns:
  • UIAWrapper

    The uia wrapper of the desktop.

__new__(backend='uia')

Singleton pattern.

Source code in automator/ui_control/inspector.py
467
468
469
470
471
472
473
474
475
476
def __new__(cls, backend: str = "uia") -> "ControlInspectorFacade":
    """
    Singleton pattern.
    """
    if backend not in cls._instances:
        instance = super().__new__(cls)
        instance.backend = backend
        instance.backend_strategy = BackendFactory.create_backend(backend)
        cls._instances[backend] = instance
    return cls._instances[backend]

find_control_elements_in_descendants(window, control_type_list=[], class_name_list=[], title_list=[], is_visible=True, is_enabled=True, depth=0)

Find control elements in descendants of the window.

Parameters:
  • window (UIAWrapper) –

    The window to find control elements.

  • control_type_list (List[str], default: [] ) –

    The control types to find.

  • class_name_list (List[str], default: [] ) –

    The class names to find.

  • title_list (List[str], default: [] ) –

    The titles to find.

  • is_visible (bool, default: True ) –

    Whether the control elements are visible.

  • is_enabled (bool, default: True ) –

    Whether the control elements are enabled.

  • depth (int, default: 0 ) –

    The depth of the descendants to find.

Returns:
  • List[UIAWrapper]

    The control elements found.

Source code in automator/ui_control/inspector.py
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
def find_control_elements_in_descendants(
    self,
    window: UIAWrapper,
    control_type_list: List[str] = [],
    class_name_list: List[str] = [],
    title_list: List[str] = [],
    is_visible: bool = True,
    is_enabled: bool = True,
    depth: int = 0,
) -> List[UIAWrapper]:
    """
    Find control elements in descendants of the window.
    :param window: The window to find control elements.
    :param control_type_list: The control types to find.
    :param class_name_list: The class names to find.
    :param title_list: The titles to find.
    :param is_visible: Whether the control elements are visible.
    :param is_enabled: Whether the control elements are enabled.
    :param depth: The depth of the descendants to find.
    :return: The control elements found.
    """
    if self.backend == "uia":
        return self.backend_strategy.find_control_elements_in_descendants(
            window, control_type_list, [], title_list, is_visible, is_enabled, depth
        )
    elif self.backend == "win32":
        return self.backend_strategy.find_control_elements_in_descendants(
            window, [], class_name_list, title_list, is_visible, is_enabled, depth
        )
    else:
        return []

get_application_root_name(window) staticmethod

Get the application name of the window.

Parameters:
  • window (UIAWrapper) –

    The window to get the application name.

Returns:
  • str

    The root application name of the window. Empty string ("") if failed to get the name.

Source code in automator/ui_control/inspector.py
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
@staticmethod
def get_application_root_name(window: UIAWrapper) -> str:
    """
    Get the application name of the window.
    :param window: The window to get the application name.
    :return: The root application name of the window. Empty string ("") if failed to get the name.
    """
    if window == None:
        return ""
    process_id = window.process_id()
    try:
        process = psutil.Process(process_id)
        return process.name()
    except psutil.NoSuchProcess:
        return ""

get_check_state(control_item) staticmethod

get the check state of the control item param control_item: the control item to get the check state return: the check state of the control item

Source code in automator/ui_control/inspector.py
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
@staticmethod
def get_check_state(control_item: auto.Control) -> bool | None:
    """
    get the check state of the control item
    param control_item: the control item to get the check state
    return: the check state of the control item
    """
    is_checked = None
    is_selected = None
    try:
        assert isinstance(
            control_item, auto.Control
        ), f"{control_item =} is not a Control"
        is_checked = (
            control_item.GetLegacyIAccessiblePattern().State
            & auto.AccessibleState.Checked
            == auto.AccessibleState.Checked
        )
        if is_checked:
            return is_checked
        is_selected = (
            control_item.GetLegacyIAccessiblePattern().State
            & auto.AccessibleState.Selected
            == auto.AccessibleState.Selected
        )
        if is_selected:
            return is_selected
        return None
    except Exception as e:
        # print(f'item {control_item} not available for check state.')
        # print(e)
        return None

get_control_info(window, field_list=[]) staticmethod

Get control info of the window.

Parameters:
  • window (UIAWrapper) –

    The window to get control info.

  • field_list (List[str], default: [] ) –

    The fields to get. return: The control info of the window.

Source code in automator/ui_control/inspector.py
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
@staticmethod
def get_control_info(
    window: UIAWrapper, field_list: List[str] = []
) -> Dict[str, str]:
    """
    Get control info of the window.
    :param window: The window to get control info.
    :param field_list: The fields to get.
    return: The control info of the window.
    """
    control_info: Dict[str, str] = {}

    def assign(prop_name: str, prop_value_func: Callable[[], str]) -> None:
        if len(field_list) > 0 and prop_name not in field_list:
            return
        control_info[prop_name] = prop_value_func()

    try:
        assign("control_type", lambda: window.element_info.control_type)
        assign("control_id", lambda: window.element_info.control_id)
        assign("control_class", lambda: window.element_info.class_name)
        assign("control_name", lambda: window.element_info.name)
        rectangle = window.element_info.rectangle
        assign(
            "control_rect",
            lambda: (
                rectangle.left,
                rectangle.top,
                rectangle.right,
                rectangle.bottom,
            ),
        )
        assign("control_text", lambda: window.element_info.name)
        assign("control_title", lambda: window.window_text())
        assign("selected", lambda: ControlInspectorFacade.get_check_state(window))

        try:
            source = window.element_info.source
            assign("source", lambda: source)
        except:
            assign("source", lambda: "")

        return control_info
    except:
        return {}

get_control_info_batch(window_list, field_list=[])

Get control info of the window.

Parameters:
  • window_list (List[UIAWrapper]) –

    The list of windows to get control info.

  • field_list (List[str], default: [] ) –

    The fields to get. return: The list of control info of the window.

Source code in automator/ui_control/inspector.py
566
567
568
569
570
571
572
573
574
575
576
577
578
def get_control_info_batch(
    self, window_list: List[UIAWrapper], field_list: List[str] = []
) -> List[Dict[str, str]]:
    """
    Get control info of the window.
    :param window_list: The list of windows to get control info.
    :param field_list: The fields to get.
    return: The list of control info of the window.
    """
    control_info_list = []
    for window in window_list:
        control_info_list.append(self.get_control_info(window, field_list))
    return control_info_list

get_control_info_list_of_dict(window_dict, field_list=[])

Get control info of the window.

Parameters:
  • window_dict (Dict[str, UIAWrapper]) –

    The dict of windows to get control info.

  • field_list (List[str], default: [] ) –

    The fields to get. return: The list of control info of the window.

Source code in automator/ui_control/inspector.py
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
def get_control_info_list_of_dict(
    self, window_dict: Dict[str, UIAWrapper], field_list: List[str] = []
) -> List[Dict[str, str]]:
    """
    Get control info of the window.
    :param window_dict: The dict of windows to get control info.
    :param field_list: The fields to get.
    return: The list of control info of the window.
    """
    control_info_list = []
    for key in window_dict.keys():
        window = window_dict[key]
        control_info = self.get_control_info(window, field_list)
        control_info["label"] = key
        control_info_list.append(control_info)
    return control_info_list

get_desktop_app_dict(remove_empty=True)

Get all the apps on the desktop and return as a dict.

Parameters:
  • remove_empty (bool, default: True ) –

    Whether to remove empty titles.

Returns:
  • Dict[str, UIAWrapper]

    The apps on the desktop as a dict.

Source code in automator/ui_control/inspector.py
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
def get_desktop_app_dict(self, remove_empty: bool = True) -> Dict[str, UIAWrapper]:
    """
    Get all the apps on the desktop and return as a dict.
    :param remove_empty: Whether to remove empty titles.
    :return: The apps on the desktop as a dict.
    """
    desktop_windows = self.get_desktop_windows(remove_empty)

    desktop_windows_with_gui = []

    for window in desktop_windows:
        try:
            window.is_normal()
            desktop_windows_with_gui.append(window)
        except:
            pass

    desktop_windows_dict = dict(
        zip(
            [str(i + 1) for i in range(len(desktop_windows_with_gui))],
            desktop_windows_with_gui,
        )
    )
    return desktop_windows_dict

get_desktop_app_info(desktop_windows_dict, field_list=['control_text', 'control_type'])

Get control info of all the apps on the desktop.

Parameters:
  • desktop_windows_dict (Dict[str, UIAWrapper]) –

    The dict of apps on the desktop.

  • field_list (List[str], default: ['control_text', 'control_type'] ) –

    The fields of app info to get.

Returns:
  • List[Dict[str, str]]

    The control info of all the apps on the desktop.

Source code in automator/ui_control/inspector.py
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
def get_desktop_app_info(
    self,
    desktop_windows_dict: Dict[str, UIAWrapper],
    field_list: List[str] = ["control_text", "control_type"],
) -> List[Dict[str, str]]:
    """
    Get control info of all the apps on the desktop.
    :param desktop_windows_dict: The dict of apps on the desktop.
    :param field_list: The fields of app info to get.
    :return: The control info of all the apps on the desktop.
    """
    desktop_windows_info = self.get_control_info_list_of_dict(
        desktop_windows_dict, field_list
    )
    return desktop_windows_info

get_desktop_windows(remove_empty=True)

Get all the apps on the desktop.

Parameters:
  • remove_empty (bool, default: True ) –

    Whether to remove empty titles.

Returns:
  • List[UIAWrapper]

    The apps on the desktop.

Source code in automator/ui_control/inspector.py
485
486
487
488
489
490
491
def get_desktop_windows(self, remove_empty: bool = True) -> List[UIAWrapper]:
    """
    Get all the apps on the desktop.
    :param remove_empty: Whether to remove empty titles.
    :return: The apps on the desktop.
    """
    return self.backend_strategy.get_desktop_windows(remove_empty)