UI Tree Logs
UFO can save the entire UI tree of the application window at every step for data collection purposes. The UI tree can represent the application's UI structure, including the window, controls, and their properties. The UI tree logs are saved in the logs/{task_name}/ui_tree
folder. You have to set the SAVE_UI_TREE
flag to True
in the config_dev.yaml
file to enable the UI tree logs. Below is an example of the UI tree logs for application:
{
"id": "node_0",
"name": "Mail - Chaoyun Zhang - Outlook",
"control_type": "Window",
"rectangle": {
"left": 628,
"top": 258,
"right": 3508,
"bottom": 1795
},
"adjusted_rectangle": {
"left": 0,
"top": 0,
"right": 2880,
"bottom": 1537
},
"relative_rectangle": {
"left": 0.0,
"top": 0.0,
"right": 1.0,
"bottom": 1.0
},
"level": 0,
"children": [
{
"id": "node_1",
"name": "",
"control_type": "Pane",
"rectangle": {
"left": 3282,
"top": 258,
"right": 3498,
"bottom": 330
},
"adjusted_rectangle": {
"left": 2654,
"top": 0,
"right": 2870,
"bottom": 72
},
"relative_rectangle": {
"left": 0.9215277777777777,
"top": 0.0,
"right": 0.9965277777777778,
"bottom": 0.0468445022771633
},
"level": 1,
"children": []
}
]
}
Fields in the UI tree logs
Below is a table of the fields in the UI tree logs:
Field |
Description |
Type |
id |
The unique identifier of the UI tree node. |
String |
name |
The name of the UI tree node. |
String |
control_type |
The type of the UI tree node. |
String |
rectangle |
The absolute position of the UI tree node. |
Dictionary |
adjusted_rectangle |
The adjusted position of the UI tree node. |
Dictionary |
relative_rectangle |
The relative position of the UI tree node. |
Dictionary |
level |
The level of the UI tree node. |
Integer |
children |
The children of the UI tree node. |
List of UI tree nodes |
Reference
A class to represent the UI tree.
Initialize the UI tree with the root element.
Parameters: |
-
root
(UIAWrapper )
–
The root element of the UI tree.
|
Source code in automator/ui_control/ui_tree.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33 | def __init__(self, root: UIAWrapper):
"""
Initialize the UI tree with the root element.
:param root: The root element of the UI tree.
"""
self.root = root
# The node counter to count the number of nodes in the UI tree.
self.node_counter = 0
try:
self._ui_tree = self._get_ui_tree(self.root)
except Exception as e:
self._ui_tree = {"error": traceback.format_exc()}
|
ui_tree: Dict[str, Any]
property
apply_ui_tree_diff(ui_tree_1, diff)
staticmethod
Apply a UI tree diff to ui_tree_1 to get ui_tree_2.
Parameters: |
-
ui_tree_1
(Dict[str, Any] )
–
-
diff
(Dict[str, Any] )
–
|
Returns: |
-
Dict[str, Any]
–
The new UI tree after applying the diff.
|
Source code in automator/ui_control/ui_tree.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323 | @staticmethod
def apply_ui_tree_diff(
ui_tree_1: Dict[str, Any], diff: Dict[str, Any]
) -> Dict[str, Any]:
"""
Apply a UI tree diff to ui_tree_1 to get ui_tree_2.
:param ui_tree_1: The original UI tree.
:param diff: The diff to apply.
:return: The new UI tree after applying the diff.
"""
ui_tree_2 = copy.deepcopy(ui_tree_1)
# Build an ID map for quick node lookups
def build_id_map(node, id_map):
id_map[node["id"]] = node
for child in node.get("children", []):
build_id_map(child, id_map)
id_map = {}
if "id" in ui_tree_2:
build_id_map(ui_tree_2, id_map)
def remove_node_by_path(path):
# The path is a list of IDs from root to target node.
# The target node is the last element. Its parent is the second to last element.
if len(path) == 1:
# Removing the root
for k in list(ui_tree_2.keys()):
del ui_tree_2[k]
id_map.clear()
return
target_id = path[-1]
parent_id = path[-2]
parent_node = id_map[parent_id]
# Find and remove the child with target_id
for i, c in enumerate(parent_node.get("children", [])):
if c["id"] == target_id:
parent_node["children"].pop(i)
break
# Remove target_id from id_map
if target_id in id_map:
del id_map[target_id]
def add_node_by_path(path, node):
# Add the node at the specified path. The parent is path[-2], the node is path[-1].
# The path[-1] should be node["id"].
if len(path) == 1:
# Replacing the root node entirely
for k in list(ui_tree_2.keys()):
del ui_tree_2[k]
for k, v in node.items():
ui_tree_2[k] = v
# Rebuild id_map
id_map.clear()
if "id" in ui_tree_2:
build_id_map(ui_tree_2, id_map)
return
target_id = path[-1]
parent_id = path[-2]
parent_node = id_map[parent_id]
# Ensure children list exists
if "children" not in parent_node:
parent_node["children"] = []
# Insert or append the node
# We don't have a numeric index anymore, we just append, assuming order doesn't matter.
# If order matters, we must store ordering info or do some heuristic.
parent_node["children"].append(node)
# Update the id_map with the newly added subtree
build_id_map(node, id_map)
def modify_node_by_path(path, changes):
# Modify fields of the node at the given ID
target_id = path[-1]
node = id_map[target_id]
for field, (old_val, new_val) in changes.items():
node[field] = new_val
# Apply removals first
# Sort removals by length of path descending so we remove deeper nodes first.
# This ensures we don't remove parents before children.
for removal in sorted(
diff["removed"], key=lambda x: len(x["path"]), reverse=True
):
remove_node_by_path(removal["path"])
# Apply additions
# Additions can be applied directly.
for addition in diff["added"]:
add_node_by_path(addition["path"], addition["node"])
# Apply modifications
for modification in diff["modified"]:
modify_node_by_path(modification["path"], modification["changes"])
return ui_tree_2
|
flatten_ui_tree()
Flatten the UI tree into a list in width-first order.
Source code in automator/ui_control/ui_tree.py
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144 | def flatten_ui_tree(self) -> List[Dict[str, Any]]:
"""
Flatten the UI tree into a list in width-first order.
"""
def flatten_tree(tree: Dict[str, Any], result: List[Dict[str, Any]]):
"""
Flatten the tree.
:param tree: The tree to flatten.
:param result: The result list.
"""
tree_info = {
"name": tree["name"],
"control_type": tree["control_type"],
"rectangle": tree["rectangle"],
"adjusted_rectangle": tree["adjusted_rectangle"],
"relative_rectangle": tree["relative_rectangle"],
"level": tree["level"],
}
result.append(tree_info)
for child in tree.get("children", []):
flatten_tree(child, result)
result = []
flatten_tree(self.ui_tree, result)
return result
|
save_ui_tree_to_json(file_path)
Save the UI tree to a JSON file.
Parameters: |
-
file_path
(str )
–
The file path to save the UI tree.
|
Source code in automator/ui_control/ui_tree.py
103
104
105
106
107
108
109
110
111
112
113
114
115 | def save_ui_tree_to_json(self, file_path: str) -> None:
"""
Save the UI tree to a JSON file.
:param file_path: The file path to save the UI tree.
"""
# Check if the file directory exists. If not, create it.
save_dir = os.path.dirname(file_path)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
with open(file_path, "w") as file:
json.dump(self.ui_tree, file, indent=4)
|
ui_tree_diff(ui_tree_1, ui_tree_2)
staticmethod
Compute the difference between two UI trees.
Parameters: |
-
ui_tree_1
(Dict[str, Any] )
–
-
ui_tree_2
(Dict[str, Any] )
–
|
Returns: |
-
–
The difference between the two UI trees.
|
Source code in automator/ui_control/ui_tree.py
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222 | @staticmethod
def ui_tree_diff(ui_tree_1: Dict[str, Any], ui_tree_2: Dict[str, Any]):
"""
Compute the difference between two UI trees.
:param ui_tree_1: The first UI tree.
:param ui_tree_2: The second UI tree.
:return: The difference between the two UI trees.
"""
diff = {"added": [], "removed": [], "modified": []}
def compare_nodes(node1, node2, path):
# Note: `path` is a list of IDs. The last element corresponds to the current node.
# If node1 doesn't exist and node2 does, it's an addition.
if node1 is None and node2 is not None:
diff["added"].append({"path": path, "node": copy.deepcopy(node2)})
return
# If node1 exists and node2 doesn't, it's a removal.
if node1 is not None and node2 is None:
diff["removed"].append({"path": path, "node": copy.deepcopy(node1)})
return
# If both don't exist, nothing to do.
if node1 is None and node2 is None:
return
# Both nodes exist, check for modifications at this node
fields_to_compare = [
"name",
"control_type",
"rectangle",
"adjusted_rectangle",
"relative_rectangle",
"level",
]
changes = {}
for field in fields_to_compare:
if node1[field] != node2[field]:
changes[field] = (node1[field], node2[field])
if changes:
diff["modified"].append({"path": path, "changes": changes})
# Compare children
children1 = node1.get("children", [])
children2 = node2.get("children", [])
# We'll assume children order is stable. If not, differences will appear as adds/removes.
max_len = max(len(children1), len(children2))
for i in range(max_len):
c1 = children1[i] if i < len(children1) else None
c2 = children2[i] if i < len(children2) else None
# Use the child's id if available from c2 (prefer new tree), else from c1
if c2 is not None:
child_id = c2["id"]
elif c1 is not None:
child_id = c1["id"]
else:
# Both None shouldn't happen since max_len ensures one must exist
child_id = "unknown_child_id"
compare_nodes(c1, c2, path + [child_id])
# Initialize the path with the root node id if it exists
if ui_tree_2 and "id" in ui_tree_2:
root_id = ui_tree_2["id"]
elif ui_tree_1 and "id" in ui_tree_1:
root_id = ui_tree_1["id"]
else:
# If no root id is present, assume a placeholder
root_id = "root"
compare_nodes(ui_tree_1, ui_tree_2, [root_id])
return diff
|
Note
Save the UI tree logs may increase the latency of the system. It is recommended to set the SAVE_UI_TREE
flag to False
when you do not need the UI tree logs.