Support modern Android emulators and Androguard 4

.gitignore

@@ -63,3 +63,5 @@ Gemfile.lock
 documents
 temp
 venv
+.venv/
+compatibility_runs/

COMPATIBILITY_REPORT.md

@@ -0,0 +1,79 @@
+# DroidBot Wikipedia compatibility report
+
+Date: 2026-07-27
+
+## Decision
+
+Conditional go for dataset development on Android APIs 29, 31, 34, and 35.
+All four emulators completed short Wikipedia exploration runs and produced
+native screenshots, accessibility-derived states, events, and `utg.js`.
+API 31 and newer emit repeated reconnect warnings from DroidBot's legacy
+accessibility helper, so multi-hour production collection is not approved
+until a soak run confirms that the helper recovers indefinitely.
+
+## Environment
+
+- Host: Windows
+- DroidBot runtime: Python 3.12.13
+- APK parser: Androguard 4.1.4
+- App: locally built Wikipedia `prodDebug`, package `org.wikipedia`
+- Emulator size: 432 x 768 at 160 dpi
+- Policy: DroidBot `dfs_greedy`
+- Event stabilization: existing fixed event interval, one second
+
+The current Wikipedia debug APK exposes both Wikipedia and LeakCanary launcher
+aliases. DroidBot now deterministically prefers a launcher owned by the target
+package, selecting `org.wikipedia.DefaultIcon`.
+
+## Results
+
+| API | Android | Events | Observations | Result | Notes |
+| --- | --- | ---: | ---: | --- | --- |
+| 29 | 10 | 10 | 11 | Pass | Touch, Back, screenshots, view trees and UTG verified; 2 ineffective transitions retained. |
+| 31 | 12 | 8 | 9 | Pass with warning | Accessibility helper reconnected repeatedly during startup. |
+| 34 | 14 | 6 | 7 | Pass with warning | Scroll and touch events succeeded; helper reconnect warnings continued. |
+| 35 | 15 | 6 | 7 | Pass after retry | First attempt encountered a transient offline ADB device; immediate retry completed. |
+
+Every completed dataset passed referential-integrity and N+1 trajectory
+validation. Every saved screenshot was 432 x 768, so no screenshot downscaling
+is required.
+
+## Compatibility changes
+
+The exploration policy, event implementations, `DeviceState` hashing, and UTG
+construction remain unchanged. Runtime compatibility required:
+
+- use the Androguard 4 import path;
+- use standard `pathlib` resource paths instead of deprecated
+  `pkg_resources`;
+- use Python 3.12 because Python 3.13 removed `telnetlib`, which DroidBot's
+  QEMU adapter still imports;
+- prefer a package-owned launcher when an APK contains launchers from debug
+  dependencies;
+- make cleanup best-effort so a temporary ADB disconnect does not mask the
+  original failure.
+
+## Stabilization decision
+
+Screenshot-difference stabilization is intentionally not implemented in the
+initial collector. Observations are captured after DroidBot's existing event
+interval. The pilot dataset should be manually audited for premature captures.
+Add image-difference stabilization only if more than 5 percent of observations
+are visibly transient or misaligned with their view trees.
+
+## Remaining go/no-go check
+
+Run at least one four-hour soak on API 29 and one on API 35. Record helper
+reconnect count, incomplete transitions, ADB disconnects, missing view trees,
+and process exit status. A run is accepted only if it exits cleanly or can
+continue through helper reconnections without losing trajectory integrity.
+
+## Run locations
+
+Smoke artifacts are under `compatibility_runs/`:
+
+- `api29_smoke_20260727`
+- `api29_collector_20260727`
+- `api31_collector_20260727`
+- `api34_collector_20260727`
+- `api35_collector_retry_20260727`

droidbot/adapter/droidbot_app.py

@@ -5,6 +5,7 @@
 import json
 import struct
 import traceback
+from pathlib import Path
 from .adapter import Adapter
 
 DROIDBOT_APP_REMOTE_ADDR = "tcp:7336"
@@ -71,8 +72,9 @@ def set_up(self):
         else:
             # install droidbot app
             try:
-                import pkg_resources
-                droidbot_app_path = pkg_resources.resource_filename("droidbot", "resources/droidbotApp.apk")
+                droidbot_app_path = str(
+                    Path(__file__).resolve().parent.parent / "resources" / "droidbotApp.apk"
+                )
                 install_cmd = ["install", droidbot_app_path]
                 self.device.adb.run_cmd(install_cmd)
                 self.logger.debug("DroidBot app installed.")

droidbot/adapter/droidbot_ime.py

@@ -2,6 +2,7 @@
 
 import logging
 import time
+from pathlib import Path
 
 from .adapter import Adapter
 
@@ -40,8 +41,9 @@ def set_up(self):
         else:
             # install droidbot app
             try:
-                import pkg_resources
-                droidbot_app_path = pkg_resources.resource_filename("droidbot", "resources/droidbotApp.apk")
+                droidbot_app_path = str(
+                    Path(__file__).resolve().parent.parent / "resources" / "droidbotApp.apk"
+                )
                 install_cmd = ["install", droidbot_app_path]
                 self.device.adb.run_cmd(install_cmd)
                 self.logger.debug("DroidBot app installed.")

droidbot/adapter/minicap.py

@@ -4,6 +4,7 @@
 import time
 import os
 from datetime import datetime
+from pathlib import Path
 from .adapter import Adapter
 
 
@@ -65,8 +66,9 @@ def set_up(self):
 
         if device is not None:
             # install minicap
-            import pkg_resources
-            local_minicap_path = pkg_resources.resource_filename("droidbot", "resources/minicap")
+            local_minicap_path = (
+                Path(__file__).resolve().parent.parent / "resources" / "minicap"
+            )
             try:
                 device.adb.shell("mkdir %s" % self.remote_minicap_path)
             except Exception:
@@ -77,9 +79,9 @@ def set_up(self):
                 minicap_bin = "minicap"
             else:
                 minicap_bin = "minicap-nopie"
-            minicap_bin_path = os.path.join(local_minicap_path, 'libs', abi, minicap_bin)
+            minicap_bin_path = os.path.join(str(local_minicap_path), 'libs', abi, minicap_bin)
             device.push_file(local_file=minicap_bin_path, remote_dir=self.remote_minicap_path)
-            minicap_so_path = os.path.join(local_minicap_path, 'jni', 'libs', f'android-{sdk}', abi, 'minicap.so')
+            minicap_so_path = os.path.join(str(local_minicap_path), 'jni', 'libs', f'android-{sdk}', abi, 'minicap.so')
             device.push_file(local_file=minicap_so_path, remote_dir=self.remote_minicap_path)
             self.logger.debug("minicap installed.")
 

droidbot/app.py

@@ -25,17 +25,30 @@ def __init__(self, app_path, output_dir=None):
             if not os.path.isdir(output_dir):
                 os.makedirs(output_dir)
 
-        from androguard.core.bytecodes.apk import APK
+        from androguard.core.apk import APK
         self.apk = APK(self.app_path)
         self.package_name = self.apk.get_package()
         self.app_name = self.apk.get_app_name()
-        self.main_activity = self.apk.get_main_activity()
+        self.main_activity = self._get_main_activity()
         self.permissions = self.apk.get_permissions()
         self.activities = self.apk.get_activities()
         self.possible_broadcasts = self.get_possible_broadcasts()
         self.dumpsys_main_activity = None
         self.hashes = self.get_hashes()
 
+    def _get_main_activity(self):
+        """Prefer a launcher owned by the target package when several exist."""
+        main_activities = self.apk.get_main_activities()
+        package_prefix = "%s." % self.package_name
+        package_activities = sorted(
+            activity
+            for activity in main_activities
+            if activity == self.package_name or activity.startswith(package_prefix)
+        )
+        if package_activities:
+            return package_activities[0]
+        return self.apk.get_main_activity()
+
     def get_package_name(self):
         """
         get package name of current app

droidbot/droidbot.py

@@ -5,8 +5,8 @@
 import logging
 import os
 import sys
-import pkg_resources
 import shutil
+from pathlib import Path
 from threading import Timer
 
 from .device import Device
@@ -58,8 +58,9 @@ def __init__(self,
         if output_dir is not None:
             if not os.path.isdir(output_dir):
                 os.makedirs(output_dir)
-            html_index_path = pkg_resources.resource_filename("droidbot", "resources/index.html")
-            stylesheets_path = pkg_resources.resource_filename("droidbot", "resources/stylesheets")
+            resources_path = Path(__file__).resolve().parent / "resources"
+            html_index_path = resources_path / "index.html"
+            stylesheets_path = resources_path / "stylesheets"
             target_stylesheets_dir = os.path.join(output_dir, "stylesheets")
             if os.path.exists(target_stylesheets_dir):
                 shutil.rmtree(target_stylesheets_dir)
@@ -176,19 +177,34 @@ def stop(self):
         self.enabled = False
         if self.timer and self.timer.is_alive():
             self.timer.cancel()
-        if self.env_manager:
-            self.env_manager.stop()
-        if self.input_manager:
-            self.input_manager.stop()
-        if self.droidbox:
-            self.droidbox.stop()
-        if self.device:
-            self.device.disconnect()
-        if not self.keep_env:
-            self.device.tear_down()
-        if not self.keep_app:
-            self.device.uninstall_app(self.app)
-        if hasattr(self.input_manager.policy, "master") and \
+        cleanup_actions = (
+            ("environment", self.env_manager.stop if self.env_manager else None),
+            ("input manager", self.input_manager.stop if self.input_manager else None),
+            ("droidbox", self.droidbox.stop if self.droidbox else None),
+            ("device connection", self.device.disconnect if self.device else None),
+            (
+                "device environment",
+                self.device.tear_down
+                if self.device and not self.keep_env
+                else None,
+            ),
+            (
+                "app installation",
+                (lambda: self.device.uninstall_app(self.app))
+                if self.device and self.app and not self.keep_app
+                else None,
+            ),
+        )
+        for label, action in cleanup_actions:
+            if action is None:
+                continue
+            try:
+                action()
+            except Exception as error:
+                self.logger.warning("Failed to clean up %s: %s", label, error)
+        if self.input_manager and \
+           self.input_manager.policy and \
+           hasattr(self.input_manager.policy, "master") and \
            self.input_manager.policy.master:
             import xmlrpc.client
             proxy = xmlrpc.client.ServerProxy(self.input_manager.policy.master)

droidbot/env_manager.py

@@ -9,6 +9,7 @@
 import json
 import time
 import os
+from pathlib import Path
 
 POLICY_NONE = "none"
 POLICY_DUMMY = "dummy"
@@ -160,10 +161,9 @@ def __init__(self, dummy_files_dir=None):
         :param: dummy_files_dir: directory to dummy files
         """
         if dummy_files_dir is None:
-            import pkg_resources
-            dummy_files_dir = pkg_resources.resource_filename("droidbot", "resources/dummy_documents")
+            dummy_files_dir = Path(__file__).resolve().parent / "resources" / "dummy_documents"
 
-        self.dummy_files_dir = dummy_files_dir
+        self.dummy_files_dir = str(dummy_files_dir)
         self.env_type = "dummy_files"
 
     def deploy(self, device):

droidbot/monitor.py

@@ -110,7 +110,7 @@ def _load_script(self, session, pid):
             self.start_time = time.clock()
 
     def _getPid(self):
-        cmd = "adb shell ps | grep " + self.packageName
+        cmd = f"adb -s {self.serial} shell ps | grep {self.packageName}"
         result = os.popen(cmd)
         if result is not None:
             return self.pid

setup.py

@@ -31,7 +31,9 @@
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
         'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
     ],
+    python_requires='>=3.9,<3.13',
     entry_points={
         'console_scripts': [
             'droidbot=start:main',
@@ -41,5 +43,5 @@
         'droidbot': [os.path.relpath(x, 'droidbot') for x in findall('droidbot/resources/')]
     },
     # androidviewclient doesnot support pip install, thus you should install it with easy_install
-    install_requires=['androguard>=3.4.0a1', 'networkx', 'Pillow'],
+    install_requires=['androguard>=4.1.4', 'networkx', 'Pillow'],
 )

tests/test_app.py

@@ -0,0 +1,44 @@
+import unittest
+
+from droidbot.app import App
+
+
+class FakeApk:
+    def __init__(self, main_activities, fallback):
+        self.main_activities = set(main_activities)
+        self.fallback = fallback
+
+    def get_main_activities(self):
+        return self.main_activities
+
+    def get_main_activity(self):
+        return self.fallback
+
+
+class AppMainActivityTest(unittest.TestCase):
+    def test_prefers_launcher_owned_by_target_package(self):
+        app = App.__new__(App)
+        app.package_name = "org.wikipedia"
+        app.apk = FakeApk(
+            {
+                "leakcanary.internal.activity.LeakLauncherActivity",
+                "org.wikipedia.DefaultIcon",
+            },
+            "leakcanary.internal.activity.LeakLauncherActivity",
+        )
+
+        self.assertEqual("org.wikipedia.DefaultIcon", app._get_main_activity())
+
+    def test_uses_androguard_fallback_without_package_launcher(self):
+        app = App.__new__(App)
+        app.package_name = "org.wikipedia"
+        app.apk = FakeApk(
+            {"external.launcher.Activity"},
+            "external.launcher.Activity",
+        )
+
+        self.assertEqual("external.launcher.Activity", app._get_main_activity())
+
+
+if __name__ == "__main__":
+    unittest.main()