GUI Automator

def annotation(
    self, params: Dict[str, str], annotation_dict: Dict[str, UIAWrapper]
) -> List[str]:
    """
    Take a screenshot of the current application window and annotate the control item on the screenshot.
    :param params: The arguments of the annotation method.
    :param annotation_dict: The dictionary of the control labels.
    """
    selected_controls_labels = params.get("control_labels", [])

    control_reannotate = [
        annotation_dict[str(label)] for label in selected_controls_labels
    ]

    return control_reannotate

def atomic_execution(self, method_name: str, params: Dict[str, Any]) -> str:
    """
    Atomic execution of the action on the control elements.
    :param method_name: The name of the method to execute.
    :param params: The arguments of the method.
    :return: The result of the action.
    """

    import traceback

    try:
        method = getattr(self.control, method_name)
        result = method(**params)
    except AttributeError:
        message = f"{self.control} doesn't have a method named {method_name}"
        print_with_color(f"Warning: {message}", "yellow")
        result = message
    except Exception as e:
        full_traceback = traceback.format_exc()
        message = f"An error occurred: {full_traceback}"
        print_with_color(f"Warning: {message}", "yellow")
        result = message
    return result

def click_input(self, params: Dict[str, Union[str, bool]]) -> str:
    """
    Click the control element.
    :param params: The arguments of the click method.
    :return: The result of the click action.
    """

    api_name = configs.get("CLICK_API", "click_input")

    if api_name == "click":
        return self.atomic_execution("click", params)
    else:
        return self.atomic_execution("click_input", params)

def click_on_coordinates(self, params: Dict[str, str]) -> str:
    """
    Click on the coordinates of the control element.
    :param params: The arguments of the click on coordinates method.
    :return: The result of the click on coordinates action.
    """

    # Get the relative coordinates fraction of the application window.
    x = float(params.get("x", 0))
    y = float(params.get("y", 0))

    button = params.get("button", "left")
    double = params.get("double", False)

    # Get the absolute coordinates of the application window.
    tranformed_x, tranformed_y = self.transform_point(x, y)

    # print(f"Clicking on {tranformed_x}, {tranformed_y}")

    self.application.set_focus()

    pyautogui.click(
        tranformed_x, tranformed_y, button=button, clicks=2 if double else 1
    )

    return ""

def drag_on_coordinates(self, params: Dict[str, str]) -> str:
    """
    Drag on the coordinates of the control element.
    :param params: The arguments of the drag on coordinates method.
    :return: The result of the drag on coordinates action.
    """

    start = self.transform_point(
        float(params.get("start_x", 0)), float(params.get("start_y", 0))
    )
    end = self.transform_point(
        float(params.get("end_x", 0)), float(params.get("end_y", 0))
    )

    duration = float(params.get("duration", 1))

    button = params.get("button", "left")

    key_hold = params.get("key_hold", None)

    self.application.set_focus()

    if key_hold:
        pyautogui.keyDown(key_hold)

    pyautogui.moveTo(start[0], start[1])
    pyautogui.dragTo(end[0], end[1], button=button, duration=duration)

    if key_hold:
        pyautogui.keyUp(key_hold)

    return ""

def key_press(self, params: Dict[str, str]) -> str:
    """
    Key press on the control element.
    :param params: The arguments of the key press method.
    :return: The result of the key press action.
    """

    keys = params.get("keys", [])

    for key in keys:
        key = key.lower()
        pyautogui.keyDown(key)
    for key in keys:
        key = key.lower()
        pyautogui.keyUp(key)

def keyboard_input(self, params: Dict[str, str]) -> str:
    """
    Keyboard input on the control element.
    :param params: The arguments of the keyboard input method.
    :return: The result of the keyboard input action.
    """

    control_focus = params.get("control_focus", True)
    keys = params.get("keys", "")
    keys = TextTransformer.transform_text(keys, "all")

    if control_focus:
        self.atomic_execution("type_keys", {"keys": keys})
    else:
        self.application.type_keys(keys=keys)
    return keys

def mouse_move(self, params: Dict[str, str]) -> str:
    """
    Mouse move on the control element.
    :param params: The arguments of the mouse move method.
    :return: The result of the mouse move action.
    """

    x = int(params.get("x", 0))
    y = int(params.get("y", 0))

    new_x, new_y = self.transform_point(x, y)

    pyautogui.moveTo(new_x, new_y, duration=0.1)

def no_action(self):
    """
    No action on the control element.
    :return: The result of the no action.
    """

    return ""

def scroll(self, params: Dict[str, str]) -> str:
    """
    Scroll on the control element.
    :param params: The arguments of the scroll method.
    :return: The result of the scroll action.
    """

    x = int(params.get("x", 0))
    y = int(params.get("y", 0))

    new_x, new_y = self.transform_point(x, y)

    scroll_x = int(params.get("scroll_x", 0))
    scroll_y = int(params.get("scroll_y", 0))

    pyautogui.vscroll(scroll_y, x=new_x, y=new_y)
    pyautogui.hscroll(scroll_x, x=new_x, y=new_y)

def set_edit_text(self, params: Dict[str, str]) -> str:
    """
    Set the edit text of the control element.
    :param params: The arguments of the set edit text method.
    :return: The result of the set edit text action.
    """

    text = params.get("text", "")
    inter_key_pause = configs.get("INPUT_TEXT_INTER_KEY_PAUSE", 0.1)

    if params.get("clear_current_text", False):
        self.control.type_keys("^a", pause=inter_key_pause)
        self.control.type_keys("{DELETE}", pause=inter_key_pause)

    if configs["INPUT_TEXT_API"] == "set_text":
        method_name = "set_edit_text"
        args = {"text": text}
    else:
        method_name = "type_keys"

        # Transform the text according to the tags.
        text = TextTransformer.transform_text(text, "all")

        args = {"keys": text, "pause": inter_key_pause, "with_spaces": True}
    try:
        result = self.atomic_execution(method_name, args)
        if (
            method_name == "set_text"
            and args["text"] not in self.control.window_text()
        ):
            raise Exception(f"Failed to use set_text: {args['text']}")
        if configs["INPUT_TEXT_ENTER"] and method_name in ["type_keys", "set_text"]:

            self.atomic_execution("type_keys", params={"keys": "{ENTER}"})
        return result
    except Exception as e:
        if method_name == "set_text":
            print_with_color(
                f"{self.control} doesn't have a method named {method_name}, trying default input method",
                "yellow",
            )
            method_name = "type_keys"
            clear_text_keys = "^a{BACKSPACE}"
            text_to_type = args["text"]
            keys_to_send = clear_text_keys + text_to_type
            method_name = "type_keys"
            args = {
                "keys": keys_to_send,
                "pause": inter_key_pause,
                "with_spaces": True,
            }
            return self.atomic_execution(method_name, args)
        else:
            return f"An error occurred: {e}"

def summary(self, params: Dict[str, str]) -> str:
    """
    Visual summary of the control element.
    :param params: The arguments of the visual summary method. should contain a key "text" with the text summary.
    :return: The result of the visual summary action.
    """

    return params.get("text")

def texts(self) -> str:
    """
    Get the text of the control element.
    :return: The text of the control element.
    """
    return self.control.texts()

def transform_point(self, fraction_x: float, fraction_y: float) -> Tuple[int, int]:
    """
    Transform the relative coordinates to the absolute coordinates.
    :param fraction_x: The relative x coordinate.
    :param fraction_y: The relative y coordinate.
    :return: The absolute coordinates.
    """
    application_rect: RECT = self.application.rectangle()
    application_x = application_rect.left
    application_y = application_rect.top
    application_width = application_rect.width()
    application_height = application_rect.height()

    x = application_x + int(application_width * fraction_x)
    y = application_y + int(application_height * fraction_y)

    return x, y

def transform_scaled_point_to_raw(
    self,
    scaled_x: int,
    scaled_y: int,
    scaled_width: int,
    scaled_height: int,
    raw_width: int,
    raw_height: int,
) -> Tuple[int, int]:
    """
    Transform the scaled coordinates to the raw coordinates.
    :param scaled_x: The scaled x coordinate.
    :param scaled_y: The scaled y coordinate.
    :param raw_width: The raw width of the application window.
    :param raw_height: The raw height of the application window.
    :param scaled_width: The scaled width of the application window.
    :param scaled_height: The scaled height of the application window.
    """

    ratio = min(scaled_width / raw_width, scaled_height / raw_height)
    raw_x = scaled_x / ratio
    raw_y = scaled_y / ratio

    return int(raw_x), int(raw_y)

def transfrom_absolute_point_to_fractional(self, x: int, y: int) -> Tuple[int, int]:
    """
    Transform the absolute coordinates to the relative coordinates.
    :param x: The absolute x coordinate on the application window.
    :param y: The absolute y coordinate on the application window.
    :return: The relative coordinates fraction.
    """
    application_rect: RECT = self.application.rectangle()
    # application_x = application_rect.left
    # application_y = application_rect.top

    application_width = application_rect.width()
    application_height = application_rect.height()

    fraction_x = x / application_width
    fraction_y = y / application_height

    return fraction_x, fraction_y

def type(self, params: Dict[str, str]) -> str:
    """
    Type on the control element.
    :param params: The arguments of the type method.
    :return: The result of the type action.
    """

    text = params.get("text", "")
    pyautogui.write(text, interval=0.1)

def wait_enabled(self, timeout: int = 10, retry_interval: int = 0.5) -> None:
    """
    Wait until the control is enabled.
    :param timeout: The timeout to wait.
    :param retry_interval: The retry interval to wait.
    """
    while not self.control.is_enabled():
        time.sleep(retry_interval)
        timeout -= retry_interval
        if timeout <= 0:
            warnings.warn(f"Timeout: {self.control} is not enabled.")
            break

def wait_visible(self, timeout: int = 10, retry_interval: int = 0.5) -> None:
    """
    Wait until the window is enabled.
    :param timeout: The timeout to wait.
    :param retry_interval: The retry interval to wait.
    """
    while not self.control.is_visible():
        time.sleep(retry_interval)
        timeout -= retry_interval
        if timeout <= 0:
            warnings.warn(f"Timeout: {self.control} is not visible.")
            break

def wheel_mouse_input(self, params: Dict[str, str]):
    """
    Wheel mouse input on the control element.
    :param params: The arguments of the wheel mouse input method.
    :return: The result of the wheel mouse input action.
    """

    if self.control is not None:
        return self.atomic_execution("wheel_mouse_input", params)
    else:
        keyboard.send_keys("{VK_CONTROL up}")
        dist = int(params.get("wheel_dist", 0))
        return self.application.wheel_mouse_input(wheel_dist=dist)