UI Tree Logs

UFO can save the entire UI tree of the application window at every step for data collection purposes. The UI tree can represent the application's UI structure, including the window, controls, and their properties. The UI tree logs are saved in the logs/{task_name}/ui_tree folder. You have to set the SAVE_UI_TREE flag to True in the config_dev.yaml file to enable the UI tree logs. Below is an example of the UI tree logs for application:

{
    "id": "node_0",
    "name": "Mail - Chaoyun Zhang - Outlook",
    "control_type": "Window",
    "rectangle": {
        "left": 628,
        "top": 258,
        "right": 3508,
        "bottom": 1795
    },
    "adjusted_rectangle": {
        "left": 0,
        "top": 0,
        "right": 2880,
        "bottom": 1537
    },
    "relative_rectangle": {
        "left": 0.0,
        "top": 0.0,
        "right": 1.0,
        "bottom": 1.0
    },
    "level": 0,
    "children": [
        {
            "id": "node_1",
            "name": "",
            "control_type": "Pane",
            "rectangle": {
                "left": 3282,
                "top": 258,
                "right": 3498,
                "bottom": 330
            },
            "adjusted_rectangle": {
                "left": 2654,
                "top": 0,
                "right": 2870,
                "bottom": 72
            },
            "relative_rectangle": {
                "left": 0.9215277777777777,
                "top": 0.0,
                "right": 0.9965277777777778,
                "bottom": 0.0468445022771633
            },
            "level": 1,
            "children": []
        }
    ]
}

Fields in the UI tree logs

Below is a table of the fields in the UI tree logs:

Field Description Type
id The unique identifier of the UI tree node. String
name The name of the UI tree node. String
control_type The type of the UI tree node. String
rectangle The absolute position of the UI tree node. Dictionary
adjusted_rectangle The adjusted position of the UI tree node. Dictionary
relative_rectangle The relative position of the UI tree node. Dictionary
level The level of the UI tree node. Integer
children The children of the UI tree node. List of UI tree nodes

Reference

A class to represent the UI tree.

Initialize the UI tree with the root element.

Parameters:
  • root (UIAWrapper) –

    The root element of the UI tree.

Source code in automator/ui_control/ui_tree.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33
def __init__(self, root: UIAWrapper):
    """
    Initialize the UI tree with the root element.
    :param root: The root element of the UI tree.
    """
    self.root = root

    # The node counter to count the number of nodes in the UI tree.
    self.node_counter = 0

    try:
        self._ui_tree = self._get_ui_tree(self.root)
    except Exception as e:
        self._ui_tree = {"error": traceback.format_exc()}

ui_tree: Dict[str, Any] property

The UI tree.

apply_ui_tree_diff(ui_tree_1, diff) staticmethod

Apply a UI tree diff to ui_tree_1 to get ui_tree_2.

Parameters:
  • ui_tree_1 (Dict[str, Any]) –

    The original UI tree.

  • diff (Dict[str, Any]) –

    The diff to apply.

Returns:
  • Dict[str, Any]

    The new UI tree after applying the diff.

Source code in automator/ui_control/ui_tree.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
@staticmethod
def apply_ui_tree_diff(
    ui_tree_1: Dict[str, Any], diff: Dict[str, Any]
) -> Dict[str, Any]:
    """
    Apply a UI tree diff to ui_tree_1 to get ui_tree_2.
    :param ui_tree_1: The original UI tree.
    :param diff: The diff to apply.
    :return: The new UI tree after applying the diff.
    """

    ui_tree_2 = copy.deepcopy(ui_tree_1)

    # Build an ID map for quick node lookups
    def build_id_map(node, id_map):
        id_map[node["id"]] = node
        for child in node.get("children", []):
            build_id_map(child, id_map)

    id_map = {}
    if "id" in ui_tree_2:
        build_id_map(ui_tree_2, id_map)

    def remove_node_by_path(path):
        # The path is a list of IDs from root to target node.
        # The target node is the last element. Its parent is the second to last element.
        if len(path) == 1:
            # Removing the root
            for k in list(ui_tree_2.keys()):
                del ui_tree_2[k]
            id_map.clear()
            return

        target_id = path[-1]
        parent_id = path[-2]
        parent_node = id_map[parent_id]
        # Find and remove the child with target_id
        for i, c in enumerate(parent_node.get("children", [])):
            if c["id"] == target_id:
                parent_node["children"].pop(i)
                break

        # Remove target_id from id_map
        if target_id in id_map:
            del id_map[target_id]

    def add_node_by_path(path, node):
        # Add the node at the specified path. The parent is path[-2], the node is path[-1].
        # The path[-1] should be node["id"].
        if len(path) == 1:
            # Replacing the root node entirely
            for k in list(ui_tree_2.keys()):
                del ui_tree_2[k]
            for k, v in node.items():
                ui_tree_2[k] = v
            # Rebuild id_map
            id_map.clear()
            if "id" in ui_tree_2:
                build_id_map(ui_tree_2, id_map)
            return

        target_id = path[-1]
        parent_id = path[-2]
        parent_node = id_map[parent_id]
        # Ensure children list exists
        if "children" not in parent_node:
            parent_node["children"] = []
        # Insert or append the node
        # We don't have a numeric index anymore, we just append, assuming order doesn't matter.
        # If order matters, we must store ordering info or do some heuristic.
        parent_node["children"].append(node)

        # Update the id_map with the newly added subtree
        build_id_map(node, id_map)

    def modify_node_by_path(path, changes):
        # Modify fields of the node at the given ID
        target_id = path[-1]
        node = id_map[target_id]
        for field, (old_val, new_val) in changes.items():
            node[field] = new_val

    # Apply removals first
    # Sort removals by length of path descending so we remove deeper nodes first.
    # This ensures we don't remove parents before children.
    for removal in sorted(
        diff["removed"], key=lambda x: len(x["path"]), reverse=True
    ):
        remove_node_by_path(removal["path"])

    # Apply additions
    # Additions can be applied directly.
    for addition in diff["added"]:
        add_node_by_path(addition["path"], addition["node"])

    # Apply modifications
    for modification in diff["modified"]:
        modify_node_by_path(modification["path"], modification["changes"])

    return ui_tree_2

flatten_ui_tree()

Flatten the UI tree into a list in width-first order.

Source code in automator/ui_control/ui_tree.py
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def flatten_ui_tree(self) -> List[Dict[str, Any]]:
    """
    Flatten the UI tree into a list in width-first order.
    """

    def flatten_tree(tree: Dict[str, Any], result: List[Dict[str, Any]]):
        """
        Flatten the tree.
        :param tree: The tree to flatten.
        :param result: The result list.
        """

        tree_info = {
            "name": tree["name"],
            "control_type": tree["control_type"],
            "rectangle": tree["rectangle"],
            "adjusted_rectangle": tree["adjusted_rectangle"],
            "relative_rectangle": tree["relative_rectangle"],
            "level": tree["level"],
        }

        result.append(tree_info)
        for child in tree.get("children", []):
            flatten_tree(child, result)

    result = []
    flatten_tree(self.ui_tree, result)
    return result

save_ui_tree_to_json(file_path)

Save the UI tree to a JSON file.

Parameters:
  • file_path (str) –

    The file path to save the UI tree.

Source code in automator/ui_control/ui_tree.py
103
104
105
106
107
108
109
110
111
112
113
114
115
def save_ui_tree_to_json(self, file_path: str) -> None:
    """
    Save the UI tree to a JSON file.
    :param file_path: The file path to save the UI tree.
    """

    # Check if the file directory exists. If not, create it.
    save_dir = os.path.dirname(file_path)
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    with open(file_path, "w") as file:
        json.dump(self.ui_tree, file, indent=4)

ui_tree_diff(ui_tree_1, ui_tree_2) staticmethod

Compute the difference between two UI trees.

Parameters:
  • ui_tree_1 (Dict[str, Any]) –

    The first UI tree.

  • ui_tree_2 (Dict[str, Any]) –

    The second UI tree.

Returns:
  • The difference between the two UI trees.

Source code in automator/ui_control/ui_tree.py
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
@staticmethod
def ui_tree_diff(ui_tree_1: Dict[str, Any], ui_tree_2: Dict[str, Any]):
    """
    Compute the difference between two UI trees.
    :param ui_tree_1: The first UI tree.
    :param ui_tree_2: The second UI tree.
    :return: The difference between the two UI trees.
    """

    diff = {"added": [], "removed": [], "modified": []}

    def compare_nodes(node1, node2, path):
        # Note: `path` is a list of IDs. The last element corresponds to the current node.
        # If node1 doesn't exist and node2 does, it's an addition.
        if node1 is None and node2 is not None:
            diff["added"].append({"path": path, "node": copy.deepcopy(node2)})
            return

        # If node1 exists and node2 doesn't, it's a removal.
        if node1 is not None and node2 is None:
            diff["removed"].append({"path": path, "node": copy.deepcopy(node1)})
            return

        # If both don't exist, nothing to do.
        if node1 is None and node2 is None:
            return

        # Both nodes exist, check for modifications at this node
        fields_to_compare = [
            "name",
            "control_type",
            "rectangle",
            "adjusted_rectangle",
            "relative_rectangle",
            "level",
        ]

        changes = {}
        for field in fields_to_compare:
            if node1[field] != node2[field]:
                changes[field] = (node1[field], node2[field])

        if changes:
            diff["modified"].append({"path": path, "changes": changes})

        # Compare children
        children1 = node1.get("children", [])
        children2 = node2.get("children", [])

        # We'll assume children order is stable. If not, differences will appear as adds/removes.
        max_len = max(len(children1), len(children2))
        for i in range(max_len):
            c1 = children1[i] if i < len(children1) else None
            c2 = children2[i] if i < len(children2) else None
            # Use the child's id if available from c2 (prefer new tree), else from c1
            if c2 is not None:
                child_id = c2["id"]
            elif c1 is not None:
                child_id = c1["id"]
            else:
                # Both None shouldn't happen since max_len ensures one must exist
                child_id = "unknown_child_id"

            compare_nodes(c1, c2, path + [child_id])

    # Initialize the path with the root node id if it exists
    if ui_tree_2 and "id" in ui_tree_2:
        root_id = ui_tree_2["id"]
    elif ui_tree_1 and "id" in ui_tree_1:
        root_id = ui_tree_1["id"]
    else:
        # If no root id is present, assume a placeholder
        root_id = "root"

    compare_nodes(ui_tree_1, ui_tree_2, [root_id])

    return diff


Note

Save the UI tree logs may increase the latency of the system. It is recommended to set the SAVE_UI_TREE flag to False when you do not need the UI tree logs.