diff --git a/.gitignore b/.gitignore index b834efc6..fb1ceca0 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,5 @@ Gemfile.lock documents temp venv +.venv/ +compatibility_runs/ diff --git a/COMPATIBILITY_REPORT.md b/COMPATIBILITY_REPORT.md new file mode 100644 index 00000000..7e202a87 --- /dev/null +++ b/COMPATIBILITY_REPORT.md @@ -0,0 +1,79 @@ +# DroidBot Wikipedia compatibility report + +Date: 2026-07-27 + +## Decision + +Conditional go for dataset development on Android APIs 29, 31, 34, and 35. +All four emulators completed short Wikipedia exploration runs and produced +native screenshots, accessibility-derived states, events, and `utg.js`. +API 31 and newer emit repeated reconnect warnings from DroidBot's legacy +accessibility helper, so multi-hour production collection is not approved +until a soak run confirms that the helper recovers indefinitely. + +## Environment + +- Host: Windows +- DroidBot runtime: Python 3.12.13 +- APK parser: Androguard 4.1.4 +- App: locally built Wikipedia `prodDebug`, package `org.wikipedia` +- Emulator size: 432 x 768 at 160 dpi +- Policy: DroidBot `dfs_greedy` +- Event stabilization: existing fixed event interval, one second + +The current Wikipedia debug APK exposes both Wikipedia and LeakCanary launcher +aliases. DroidBot now deterministically prefers a launcher owned by the target +package, selecting `org.wikipedia.DefaultIcon`. + +## Results + +| API | Android | Events | Observations | Result | Notes | +| --- | --- | ---: | ---: | --- | --- | +| 29 | 10 | 10 | 11 | Pass | Touch, Back, screenshots, view trees and UTG verified; 2 ineffective transitions retained. | +| 31 | 12 | 8 | 9 | Pass with warning | Accessibility helper reconnected repeatedly during startup. | +| 34 | 14 | 6 | 7 | Pass with warning | Scroll and touch events succeeded; helper reconnect warnings continued. | +| 35 | 15 | 6 | 7 | Pass after retry | First attempt encountered a transient offline ADB device; immediate retry completed. | + +Every completed dataset passed referential-integrity and N+1 trajectory +validation. Every saved screenshot was 432 x 768, so no screenshot downscaling +is required. + +## Compatibility changes + +The exploration policy, event implementations, `DeviceState` hashing, and UTG +construction remain unchanged. Runtime compatibility required: + +- use the Androguard 4 import path; +- use standard `pathlib` resource paths instead of deprecated + `pkg_resources`; +- use Python 3.12 because Python 3.13 removed `telnetlib`, which DroidBot's + QEMU adapter still imports; +- prefer a package-owned launcher when an APK contains launchers from debug + dependencies; +- make cleanup best-effort so a temporary ADB disconnect does not mask the + original failure. + +## Stabilization decision + +Screenshot-difference stabilization is intentionally not implemented in the +initial collector. Observations are captured after DroidBot's existing event +interval. The pilot dataset should be manually audited for premature captures. +Add image-difference stabilization only if more than 5 percent of observations +are visibly transient or misaligned with their view trees. + +## Remaining go/no-go check + +Run at least one four-hour soak on API 29 and one on API 35. Record helper +reconnect count, incomplete transitions, ADB disconnects, missing view trees, +and process exit status. A run is accepted only if it exits cleanly or can +continue through helper reconnections without losing trajectory integrity. + +## Run locations + +Smoke artifacts are under `compatibility_runs/`: + +- `api29_smoke_20260727` +- `api29_collector_20260727` +- `api31_collector_20260727` +- `api34_collector_20260727` +- `api35_collector_retry_20260727` diff --git a/droidbot/adapter/droidbot_app.py b/droidbot/adapter/droidbot_app.py index 83fd93d0..be3f4f41 100644 --- a/droidbot/adapter/droidbot_app.py +++ b/droidbot/adapter/droidbot_app.py @@ -5,6 +5,7 @@ import json import struct import traceback +from pathlib import Path from .adapter import Adapter DROIDBOT_APP_REMOTE_ADDR = "tcp:7336" @@ -71,8 +72,9 @@ def set_up(self): else: # install droidbot app try: - import pkg_resources - droidbot_app_path = pkg_resources.resource_filename("droidbot", "resources/droidbotApp.apk") + droidbot_app_path = str( + Path(__file__).resolve().parent.parent / "resources" / "droidbotApp.apk" + ) install_cmd = ["install", droidbot_app_path] self.device.adb.run_cmd(install_cmd) self.logger.debug("DroidBot app installed.") diff --git a/droidbot/adapter/droidbot_ime.py b/droidbot/adapter/droidbot_ime.py index 4e6fe250..88b102bc 100644 --- a/droidbot/adapter/droidbot_ime.py +++ b/droidbot/adapter/droidbot_ime.py @@ -2,6 +2,7 @@ import logging import time +from pathlib import Path from .adapter import Adapter @@ -40,8 +41,9 @@ def set_up(self): else: # install droidbot app try: - import pkg_resources - droidbot_app_path = pkg_resources.resource_filename("droidbot", "resources/droidbotApp.apk") + droidbot_app_path = str( + Path(__file__).resolve().parent.parent / "resources" / "droidbotApp.apk" + ) install_cmd = ["install", droidbot_app_path] self.device.adb.run_cmd(install_cmd) self.logger.debug("DroidBot app installed.") diff --git a/droidbot/adapter/minicap.py b/droidbot/adapter/minicap.py index 86c15095..a5bf56f6 100644 --- a/droidbot/adapter/minicap.py +++ b/droidbot/adapter/minicap.py @@ -4,6 +4,7 @@ import time import os from datetime import datetime +from pathlib import Path from .adapter import Adapter @@ -65,8 +66,9 @@ def set_up(self): if device is not None: # install minicap - import pkg_resources - local_minicap_path = pkg_resources.resource_filename("droidbot", "resources/minicap") + local_minicap_path = ( + Path(__file__).resolve().parent.parent / "resources" / "minicap" + ) try: device.adb.shell("mkdir %s" % self.remote_minicap_path) except Exception: @@ -77,9 +79,9 @@ def set_up(self): minicap_bin = "minicap" else: minicap_bin = "minicap-nopie" - minicap_bin_path = os.path.join(local_minicap_path, 'libs', abi, minicap_bin) + minicap_bin_path = os.path.join(str(local_minicap_path), 'libs', abi, minicap_bin) device.push_file(local_file=minicap_bin_path, remote_dir=self.remote_minicap_path) - minicap_so_path = os.path.join(local_minicap_path, 'jni', 'libs', f'android-{sdk}', abi, 'minicap.so') + minicap_so_path = os.path.join(str(local_minicap_path), 'jni', 'libs', f'android-{sdk}', abi, 'minicap.so') device.push_file(local_file=minicap_so_path, remote_dir=self.remote_minicap_path) self.logger.debug("minicap installed.") diff --git a/droidbot/app.py b/droidbot/app.py index dfb25795..cf17800d 100644 --- a/droidbot/app.py +++ b/droidbot/app.py @@ -25,17 +25,30 @@ def __init__(self, app_path, output_dir=None): if not os.path.isdir(output_dir): os.makedirs(output_dir) - from androguard.core.bytecodes.apk import APK + from androguard.core.apk import APK self.apk = APK(self.app_path) self.package_name = self.apk.get_package() self.app_name = self.apk.get_app_name() - self.main_activity = self.apk.get_main_activity() + self.main_activity = self._get_main_activity() self.permissions = self.apk.get_permissions() self.activities = self.apk.get_activities() self.possible_broadcasts = self.get_possible_broadcasts() self.dumpsys_main_activity = None self.hashes = self.get_hashes() + def _get_main_activity(self): + """Prefer a launcher owned by the target package when several exist.""" + main_activities = self.apk.get_main_activities() + package_prefix = "%s." % self.package_name + package_activities = sorted( + activity + for activity in main_activities + if activity == self.package_name or activity.startswith(package_prefix) + ) + if package_activities: + return package_activities[0] + return self.apk.get_main_activity() + def get_package_name(self): """ get package name of current app diff --git a/droidbot/droidbot.py b/droidbot/droidbot.py index b604ea6f..4c38bbe1 100644 --- a/droidbot/droidbot.py +++ b/droidbot/droidbot.py @@ -5,8 +5,8 @@ import logging import os import sys -import pkg_resources import shutil +from pathlib import Path from threading import Timer from .device import Device @@ -58,8 +58,9 @@ def __init__(self, if output_dir is not None: if not os.path.isdir(output_dir): os.makedirs(output_dir) - html_index_path = pkg_resources.resource_filename("droidbot", "resources/index.html") - stylesheets_path = pkg_resources.resource_filename("droidbot", "resources/stylesheets") + resources_path = Path(__file__).resolve().parent / "resources" + html_index_path = resources_path / "index.html" + stylesheets_path = resources_path / "stylesheets" target_stylesheets_dir = os.path.join(output_dir, "stylesheets") if os.path.exists(target_stylesheets_dir): shutil.rmtree(target_stylesheets_dir) @@ -176,19 +177,34 @@ def stop(self): self.enabled = False if self.timer and self.timer.is_alive(): self.timer.cancel() - if self.env_manager: - self.env_manager.stop() - if self.input_manager: - self.input_manager.stop() - if self.droidbox: - self.droidbox.stop() - if self.device: - self.device.disconnect() - if not self.keep_env: - self.device.tear_down() - if not self.keep_app: - self.device.uninstall_app(self.app) - if hasattr(self.input_manager.policy, "master") and \ + cleanup_actions = ( + ("environment", self.env_manager.stop if self.env_manager else None), + ("input manager", self.input_manager.stop if self.input_manager else None), + ("droidbox", self.droidbox.stop if self.droidbox else None), + ("device connection", self.device.disconnect if self.device else None), + ( + "device environment", + self.device.tear_down + if self.device and not self.keep_env + else None, + ), + ( + "app installation", + (lambda: self.device.uninstall_app(self.app)) + if self.device and self.app and not self.keep_app + else None, + ), + ) + for label, action in cleanup_actions: + if action is None: + continue + try: + action() + except Exception as error: + self.logger.warning("Failed to clean up %s: %s", label, error) + if self.input_manager and \ + self.input_manager.policy and \ + hasattr(self.input_manager.policy, "master") and \ self.input_manager.policy.master: import xmlrpc.client proxy = xmlrpc.client.ServerProxy(self.input_manager.policy.master) diff --git a/droidbot/env_manager.py b/droidbot/env_manager.py index 79faaf86..121ab9e3 100644 --- a/droidbot/env_manager.py +++ b/droidbot/env_manager.py @@ -9,6 +9,7 @@ import json import time import os +from pathlib import Path POLICY_NONE = "none" POLICY_DUMMY = "dummy" @@ -160,10 +161,9 @@ def __init__(self, dummy_files_dir=None): :param: dummy_files_dir: directory to dummy files """ if dummy_files_dir is None: - import pkg_resources - dummy_files_dir = pkg_resources.resource_filename("droidbot", "resources/dummy_documents") + dummy_files_dir = Path(__file__).resolve().parent / "resources" / "dummy_documents" - self.dummy_files_dir = dummy_files_dir + self.dummy_files_dir = str(dummy_files_dir) self.env_type = "dummy_files" def deploy(self, device): diff --git a/droidbot/monitor.py b/droidbot/monitor.py index 311607f7..c95894d7 100644 --- a/droidbot/monitor.py +++ b/droidbot/monitor.py @@ -110,7 +110,7 @@ def _load_script(self, session, pid): self.start_time = time.clock() def _getPid(self): - cmd = "adb shell ps | grep " + self.packageName + cmd = f"adb -s {self.serial} shell ps | grep {self.packageName}" result = os.popen(cmd) if result is not None: return self.pid diff --git a/setup.py b/setup.py index 2a18c5c6..10c903cf 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,9 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. 'Programming Language :: Python', + 'Programming Language :: Python :: 3', ], + python_requires='>=3.9,<3.13', entry_points={ 'console_scripts': [ 'droidbot=start:main', @@ -41,5 +43,5 @@ 'droidbot': [os.path.relpath(x, 'droidbot') for x in findall('droidbot/resources/')] }, # androidviewclient doesnot support pip install, thus you should install it with easy_install - install_requires=['androguard>=3.4.0a1', 'networkx', 'Pillow'], + install_requires=['androguard>=4.1.4', 'networkx', 'Pillow'], ) diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 00000000..012c6fb1 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,44 @@ +import unittest + +from droidbot.app import App + + +class FakeApk: + def __init__(self, main_activities, fallback): + self.main_activities = set(main_activities) + self.fallback = fallback + + def get_main_activities(self): + return self.main_activities + + def get_main_activity(self): + return self.fallback + + +class AppMainActivityTest(unittest.TestCase): + def test_prefers_launcher_owned_by_target_package(self): + app = App.__new__(App) + app.package_name = "org.wikipedia" + app.apk = FakeApk( + { + "leakcanary.internal.activity.LeakLauncherActivity", + "org.wikipedia.DefaultIcon", + }, + "leakcanary.internal.activity.LeakLauncherActivity", + ) + + self.assertEqual("org.wikipedia.DefaultIcon", app._get_main_activity()) + + def test_uses_androguard_fallback_without_package_launcher(self): + app = App.__new__(App) + app.package_name = "org.wikipedia" + app.apk = FakeApk( + {"external.launcher.Activity"}, + "external.launcher.Activity", + ) + + self.assertEqual("external.launcher.Activity", app._get_main_activity()) + + +if __name__ == "__main__": + unittest.main()