Automation

[johnmalek312]

@claude look into how i can use droidrun tools for automation.

I was thinking like this ```py
import asyncio
from droidrun.tools import AdbTools

async def main():
# Initialize ADB tools (auto-detect device)
tools = AdbTools()

# -------------------------
#  Your automation steps here
#  Example:
#  await tools.tap_by_index(5)
#  await tools.sleep(1)
#  await tools.swipe(100, 500, 100, 100)
# -------------------------

pass  # remove this when adding your own code

Entry point

if name == "main":
asyncio.run(main())```

tap_by_index isnt very good for automation, look into how i can use tap by coordinates and how i can do text search then use the taps, etc..., give me some ideas of how i can utilise droidrun for testing and automation without the agent. make it a bit comprehensive, make sure its accurate and look.

[claude]

Claude finished @johnmalek312's task —— View job

---

DroidRun Automation Guide - Using Tools Without the Agent

I've analyzed the DroidRun codebase and found comprehensive automation capabilities beyond tap_by_index. Here's how you can utilize DroidRun for testing and automation:

---

🎯 Core Automation Methods

1. Coordinate-Based Tapping

You can tap at specific screen coordinates using tap_by_coordinates():

import asyncio
from droidrun.tools import AdbTools

async def coordinate_based_automation():
    tools = AdbTools()
    await tools.connect()
    
    # Tap at specific coordinates (x, y)
    await tools.tap_by_coordinates(540, 960)
    
    # Multiple taps at different locations
    coordinates = [(100, 200), (300, 400), (500, 600)]
    for x, y in coordinates:
        await tools.tap_by_coordinates(x, y)
        await asyncio.sleep(0.5)  # Wait between taps

asyncio.run(coordinate_based_automation())

2. Element Search with Accessibility Tree

The most powerful approach is using get_state() to retrieve the UI hierarchy and search for elements:

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters

async def search_and_tap():
    tools = AdbTools()
    await tools.connect()
    
    # Get the current UI state
    formatted_text, focused_text, a11y_tree, phone_state = await tools.get_state()
    
    # The a11y_tree contains all UI elements with their properties
    # You can search through it to find elements by text, ID, or other attributes
    
    # Example: Find an element by searching the tree
    for element in a11y_tree:
        if element.get('text') == 'Login':
            # Get element index and tap it
            index = element.get('index')
            await tools.tap_by_index(index)
            break

asyncio.run(search_and_tap())

3. Advanced Text Search with Filters

DroidRun has a powerful Filters class for complex element searches:

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters, get_element_center

async def advanced_text_search():
    tools = AdbTools()
    await tools.connect()
    
    # Get raw tree from the accessibility tree cache
    formatted_text, focused_text, a11y_tree, phone_state = await tools.get_state()
    raw_tree = tools.raw_tree_cache
    
    # Search for elements by text (case-insensitive)
    text_filter = Filters.text_matches("Login")
    matching_elements = text_filter([raw_tree])
    
    if matching_elements:
        element = matching_elements[0]
        # Get center coordinates
        x, y = get_element_center(element)
        await tools.tap_by_coordinates(x, y)
    
    # Search by resource ID
    id_filter = Filters.id_matches("submit_button")
    buttons = id_filter([raw_tree])
    
    # Search for clickable elements only
    clickable_filter = Filters.clickable()
    clickable_elements = clickable_filter([raw_tree])
    
    # Combine filters: find clickable elements with specific text
    combined_filter = Filters.compose([
        Filters.text_matches("Submit"),
        Filters.clickable()
    ])
    results = combined_filter([raw_tree])
    
    # Spatial search: find element below another
    below_filter = Filters.below(Filters.text_matches("Username"))
    elements_below = below_filter([raw_tree])

asyncio.run(advanced_text_search())

4. Swipe Gestures

Perform swipes for scrolling or gestures:

import asyncio
from droidrun.tools import AdbTools

async def swipe_automation():
    tools = AdbTools()
    await tools.connect()
    
    # Swipe from (start_x, start_y) to (end_x, end_y)
    # duration_ms controls the speed
    
    # Scroll down
    await tools.swipe(540, 1500, 540, 500, duration_ms=500)
    
    # Scroll up
    await tools.swipe(540, 500, 540, 1500, duration_ms=500)
    
    # Horizontal swipe (left to right)
    await tools.swipe(100, 960, 900, 960, duration_ms=300)
    
    # Long press (swipe with same start/end coordinates)
    await tools.swipe(540, 960, 540, 960, duration_ms=2000)

asyncio.run(swipe_automation())

5. Text Input

Input text into fields:

import asyncio
from droidrun.tools import AdbTools

async def text_input_automation():
    tools = AdbTools()
    await tools.connect()
    
    # First, get the UI state to find the input field
    formatted_text, focused_text, a11y_tree, phone_state = await tools.get_state()
    
    # Find the input field by index and tap it to focus
    input_field_index = 5  # Replace with actual index
    await tools.tap_by_index(input_field_index)
    
    # Input text (with optional clear)
    await tools.input_text("username@example.com", clear=True)
    
    # Or input to specific element by index
    await tools.input_text("password123", index=6, clear=True)
    
    # Press Enter key
    await tools.press_key(66)  # 66 = ENTER keycode

asyncio.run(text_input_automation())

6. Key Press Events

Simulate hardware key presses:

import asyncio
from droidrun.tools import AdbTools

async def key_press_automation():
    tools = AdbTools()
    await tools.connect()
    
    # Common keycodes:
    await tools.press_key(3)   # HOME
    await tools.press_key(4)   # BACK (or use tools.back())
    await tools.press_key(66)  # ENTER
    await tools.press_key(67)  # DELETE
    
    # Go back using the dedicated method
    await tools.back()

asyncio.run(key_press_automation())

7. App Management

Start apps and manage packages:

import asyncio
from droidrun.tools import AdbTools

async def app_management():
    tools = AdbTools()
    await tools.connect()
    
    # Start an app by package name (activity is auto-detected)
    await tools.start_app("com.android.settings")
    
    # Start with specific activity
    await tools.start_app("com.android.settings", ".Settings")
    
    # List installed packages (user apps only)
    packages = await tools.list_packages(include_system_apps=False)
    print(f"Found {len(packages)} user apps")
    
    # Get apps with human-readable names
    apps = await tools.get_apps(include_system=False)
    for app in apps:
        print(f"{app['label']}: {app['package']}")
    
    # Install an APK
    await tools.install_app("/path/to/app.apk", reinstall=True, grant_permissions=True)

asyncio.run(app_management())

8. Screenshots

Capture device screenshots:

import asyncio
from droidrun.tools import AdbTools

async def screenshot_automation():
    tools = AdbTools()
    await tools.connect()
    
    # Take screenshot
    format_type, image_bytes = await tools.take_screenshot()
    
    # Save to file
    with open("screenshot.png", "wb") as f:
        f.write(image_bytes)
    
    # Take screenshot without Portal overlay
    format_type, image_bytes = await tools.take_screenshot(hide_overlay=True)

asyncio.run(screenshot_automation())

---

🔥 Complete Automation Examples

Example 1: Login Flow Automation

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters, get_element_center

async def login_flow():
    """Automate a login flow with text search and input."""
    tools = AdbTools()
    await tools.connect()
    
    # Start the app
    await tools.start_app("com.example.app")
    await asyncio.sleep(2)
    
    # Get UI state
    _, _, a11y_tree, _ = await tools.get_state()
    raw_tree = tools.raw_tree_cache
    
    # Find and tap username field
    username_filter = Filters.compose([
        Filters.text_matches("Username"),
        Filters.clickable()
    ])
    username_elements = username_filter([raw_tree])
    
    if username_elements:
        x, y = get_element_center(username_elements[0])
        await tools.tap_by_coordinates(x, y)
        await asyncio.sleep(0.5)
        await tools.input_text("user@example.com", clear=True)
    
    # Find password field (below username)
    password_filter = Filters.below(Filters.text_matches("Username"))
    password_elements = password_filter([raw_tree])
    
    if password_elements:
        x, y = get_element_center(password_elements[0])
        await tools.tap_by_coordinates(x, y)
        await asyncio.sleep(0.5)
        await tools.input_text("SecurePass123", clear=True)
    
    # Find and tap login button
    login_filter = Filters.compose([
        Filters.text_matches("Login"),
        Filters.clickable()
    ])
    login_buttons = login_filter([raw_tree])
    
    if login_buttons:
        x, y = get_element_center(login_buttons[0])
        await tools.tap_by_coordinates(x, y)
    
    # Wait and verify login
    await asyncio.sleep(3)
    _, _, new_tree, _ = await tools.get_state()

asyncio.run(login_flow())

Example 2: Scrolling and List Interaction

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters, get_element_center

async def scroll_and_interact():
    """Scroll through a list and interact with items."""
    tools = AdbTools()
    await tools.connect()
    
    # Get screen dimensions from device
    _, _, _, phone_state = await tools.get_state()
    screen_width = phone_state.get('screen_width', 1080)
    screen_height = phone_state.get('screen_height', 1920)
    
    center_x = screen_width // 2
    
    for i in range(5):  # Scroll 5 times
        # Scroll down
        await tools.swipe(
            center_x, screen_height * 3 // 4,  # Start at 75% height
            center_x, screen_height * 1 // 4,  # End at 25% height
            duration_ms=500
        )
        
        await asyncio.sleep(1)
        
        # Get current state after scroll
        _, _, a11y_tree, _ = await tools.get_state()
        raw_tree = tools.raw_tree_cache
        
        # Search for specific item
        item_filter = Filters.text_matches("Target Item")
        items = item_filter([raw_tree])
        
        if items:
            x, y = get_element_center(items[0])
            await tools.tap_by_coordinates(x, y)
            print("Found and tapped target item!")
            break

asyncio.run(scroll_and_interact())

Example 3: Form Filling with Validation

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters, get_element_center

async def fill_form():
    """Fill out a multi-field form with validation."""
    tools = AdbTools()
    await tools.connect()
    
    form_data = {
        "Name": "John Doe",
        "Email": "john@example.com",
        "Phone": "555-1234",
        "Address": "123 Main St"
    }
    
    for field_label, value in form_data.items():
        # Get fresh UI state
        _, _, a11y_tree, _ = await tools.get_state()
        raw_tree = tools.raw_tree_cache
        
        # Find field by label
        field_filter = Filters.text_matches(field_label)
        fields = field_filter([raw_tree])
        
        if fields:
            # Get input field (usually below or after the label)
            input_filter = Filters.below(field_filter)
            inputs = input_filter([raw_tree])
            
            if inputs:
                x, y = get_element_center(inputs[0])
                await tools.tap_by_coordinates(x, y)
                await asyncio.sleep(0.3)
                await tools.input_text(value, clear=True)
                await asyncio.sleep(0.3)
        
        # Take screenshot for verification
        _, screenshot = await tools.take_screenshot()
        with open(f"form_{field_label}.png", "wb") as f:
            f.write(screenshot)
    
    # Submit form
    _, _, a11y_tree, _ = await tools.get_state()
    raw_tree = tools.raw_tree_cache
    
    submit_filter = Filters.compose([
        Filters.text_matches("Submit"),
        Filters.clickable()
    ])
    submit_buttons = submit_filter([raw_tree])
    
    if submit_buttons:
        x, y = get_element_center(submit_buttons[0])
        await tools.tap_by_coordinates(x, y)

asyncio.run(fill_form())

Example 4: Testing with Assertions

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters

async def test_app_workflow():
    """Test an app workflow with assertions."""
    tools = AdbTools()
    await tools.connect()
    
    # Test 1: App launches successfully
    await tools.start_app("com.example.app")
    await asyncio.sleep(2)
    
    _, _, a11y_tree, _ = await tools.get_state()
    raw_tree = tools.raw_tree_cache
    
    # Assert welcome screen appears
    welcome_filter = Filters.text_matches("Welcome")
    welcome_elements = welcome_filter([raw_tree])
    assert len(welcome_elements) > 0, "Welcome screen not found"
    print("✓ Test 1 passed: App launched successfully")
    
    # Test 2: Navigation works
    menu_filter = Filters.compose([
        Filters.id_matches("menu_button"),
        Filters.clickable()
    ])
    menu_buttons = menu_filter([raw_tree])
    
    if menu_buttons:
        from droidrun.tools.element_search import get_element_center
        x, y = get_element_center(menu_buttons[0])
        await tools.tap_by_coordinates(x, y)
        await asyncio.sleep(1)
        
        # Verify menu opened
        _, _, new_tree, _ = await tools.get_state()
        new_raw = tools.raw_tree_cache
        
        menu_items = Filters.text_matches("Settings")([new_raw])
        assert len(menu_items) > 0, "Menu did not open"
        print("✓ Test 2 passed: Navigation works")
    
    # Test 3: Back button works
    await tools.back()
    await asyncio.sleep(1)
    
    _, _, back_tree, _ = await tools.get_state()
    back_raw = tools.raw_tree_cache
    
    home_elements = Filters.text_matches("Welcome")([back_raw])
    assert len(home_elements) > 0, "Back navigation failed"
    print("✓ Test 3 passed: Back button works")
    
    print("\n✅ All tests passed!")

asyncio.run(test_app_workflow())

---

📋 Best Practices

1. Always call connect() first - Establish connection before any operations
2. Use get_state() before searching - Get fresh UI state to ensure accuracy
3. Add sleep delays - Give UI time to update between actions (await asyncio.sleep(0.5))
4. Use Filters for robust searches - More reliable than hardcoded indices
5. Take screenshots for debugging - Capture state at key points
6. Handle exceptions - Wrap automation in try/except blocks
7. Verify state changes - Check that actions had the expected effect
8. Use coordinate taps when indices change - More stable for dynamic UIs

---

🎓 Available Filter Methods

The Filters class provides powerful search capabilities:

• Text matching: text_matches(), id_matches()
• Spatial: below(), above(), left_of(), right_of()
• Traits: clickable(), enabled(), focused(), checked()
• Hierarchy: contains_child(), contains_descendants(), child_of()
• Composition: compose() (pipeline), intersect() (AND logic)
• Selection: index(), clickable_first(), deepest_matching()

---

This comprehensive guide should give you everything you need to use DroidRun for automation and testing without the agent! The key is combining get_state() for UI discovery with the Filters class for robust element search, then using coordinate-based taps for reliable interaction.

[iuy123]

下载大小是多少

[johnmalek312]

@claude write a function that would allow clicking by resource id. {'index': 34, 'resourceId': 'com.google.android.apps.nbu.files:id/move_to_trash_action', 'className': 'Button', 'text': 'Placer dans la corbeille', 'bounds': '601,298,667,364', 'children': []},

[claude]

Claude finished @johnmalek312's task —— View job • claude/issue-223-20251205-2230

---

Added tap_by_resource_id() Function ✅

I've successfully implemented a new method tap_by_resource_id() in the AdbTools class that allows you to click elements by their resource ID!

🎯 How to Use

import asyncio
from droidrun.tools import AdbTools

async def main():
    tools = AdbTools()
    await tools.connect()
    
    # Get the current UI state (required before tapping)
    await tools.get_state()
    
    # Tap by resource ID - supports both full and short IDs
    # Full ID
    await tools.tap_by_resource_id("com.google.android.apps.nbu.files:id/move_to_trash_action")
    
    # Or use just the short ID
    await tools.tap_by_resource_id("move_to_trash_action")
    
asyncio.run(main())

📋 Key Features

• Flexible ID matching: Accepts both full resource IDs (e.g., "com.example.app:id/button") and short IDs (e.g., "button")
• Robust search: Uses the existing Filters.id_matches() for reliable element finding
• Descriptive feedback: Returns detailed information about the tapped element
• Error handling: Clear error messages when elements aren't found
• Trajectory tracking: Integrates with DroidRun's event recording system

🔍 How It Works

1. The method searches through the raw accessibility tree cache using the resource ID filter
2. Finds the first matching element
3. Calculates the center coordinates using get_element_center()
4. Taps at those coordinates
5. Returns a descriptive message with element details

📝 Example Output

When successful, you'll see output like:

Tapped element with resource ID 'move_to_trash_action' | Text: 'Placer dans la corbeille' | Class: Button | Clickable: True | Coordinates: (634, 331)

⚠️ Important Notes

• Always call get_state() before using tap_by_resource_id() to ensure the UI tree cache is up to date
• If multiple elements have the same resource ID, the first one found will be tapped
• The method works with the raw accessibility tree, so it can find elements even if they're not clickable (though you should generally tap clickable elements)

🚀 Complete Automation Example

import asyncio
from droidrun.tools import AdbTools

async def automation_with_resource_ids():
    tools = AdbTools()
    await tools.connect()
    
    # Start your app
    await tools.start_app("com.google.android.apps.nbu.files")
    await asyncio.sleep(2)
    
    # Get UI state
    await tools.get_state()
    
    # Interact with elements by resource ID
    await tools.tap_by_resource_id("search_button")
    await asyncio.sleep(0.5)
    
    await tools.input_text("My Documents", clear=True)
    await asyncio.sleep(0.5)
    
    # Refresh state after UI changes
    await tools.get_state()
    
    # Tap another element
    await tools.tap_by_resource_id("move_to_trash_action")
    
asyncio.run(automation_with_resource_ids())

---

Create PR |

[johnmalek312]

@claude i dont want it in a pr, i want a function to tap by resource on top of droidrun. Show me basic example

[claude]

Claude finished @johnmalek312's task —— View job

---

Basic Example: Tap by Resource ID

Here's a simple standalone function you can use on top of DroidRun to tap elements by their resource ID:

import asyncio
from droidrun.tools import AdbTools
from droidrun.tools.element_search import Filters, get_element_center


async def tap_by_resource_id(tools: AdbTools, resource_id: str) -> bool:
    """
    Tap an element by its resource ID.
    
    Args:
        tools: AdbTools instance (must be connected)
        resource_id: Resource ID to search for (e.g., "move_to_trash_action" or full ID)
    
    Returns:
        True if element was found and tapped, False otherwise
    """
    # Get fresh UI state
    await tools.get_state()
    
    # Get the raw tree from cache
    raw_tree = tools.raw_tree_cache
    
    if not raw_tree:
        print("Error: No UI tree available. Call get_state() first.")
        return False
    
    # Search for element by resource ID
    id_filter = Filters.id_matches(resource_id)
    matching_elements = id_filter([raw_tree])
    
    if not matching_elements:
        print(f"No element found with resource ID: {resource_id}")
        return False
    
    # Get the first matching element
    element = matching_elements[0]
    
    # Get center coordinates
    x, y = get_element_center(element)
    
    # Tap at coordinates
    await tools.tap_by_coordinates(x, y)
    
    # Print confirmation
    text = element.get('text', '')
    class_name = element.get('className', '')
    print(f"✓ Tapped '{text}' ({class_name}) at ({x}, {y})")
    
    return True


# Example usage
async def main():
    tools = AdbTools()
    await tools.connect()
    
    # Example 1: Using short resource ID
    await tap_by_resource_id(tools, "move_to_trash_action")
    
    # Example 2: Using full resource ID
    await tap_by_resource_id(tools, "com.google.android.apps.nbu.files:id/move_to_trash_action")
    
    # Example 3: Search by text, then tap
    await tools.get_state()
    raw_tree = tools.raw_tree_cache
    
    text_filter = Filters.text_matches("Placer dans la corbeille")
    elements = text_filter([raw_tree])
    
    if elements:
        x, y = get_element_center(elements[0])
        await tools.tap_by_coordinates(x, y)
        print(f"✓ Tapped button at ({x}, {y})")


if __name__ == "__main__":
    asyncio.run(main())

Key Points:

1. Always call get_state() first - This populates the raw_tree_cache needed for searching
2. Use Filters.id_matches() - Works with both short IDs ("move_to_trash_action") and full IDs ("com.google.android.apps.nbu.files:id/move_to_trash_action")
3. Get coordinates with get_element_center() - Calculates the center point from the element's bounds
4. Tap using tap_by_coordinates() - More reliable than index-based tapping

Alternative: Search by Text

from droidrun.tools.element_search import Filters, get_element_center

async def tap_by_text(tools: AdbTools, text: str) -> bool:
    """Tap an element by its text content."""
    await tools.get_state()
    raw_tree = tools.raw_tree_cache
    
    text_filter = Filters.text_matches(text)
    elements = text_filter([raw_tree])
    
    if elements:
        x, y = get_element_center(elements[0])
        await tools.tap_by_coordinates(x, y)
        print(f"✓ Tapped '{text}' at ({x}, {y})")
        return True
    
    print(f"✗ Element with text '{text}' not found")
    return False

This approach is much more reliable than tap_by_index for automation since resource IDs and text rarely change, while indices can shift with UI updates!

---

[notkojak]

@claude write a function that would stop_app as same as start_app, give me an exemple