mohalmah commited on
Commit
7ee719a
Β·
verified Β·
1 Parent(s): c56758c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +348 -395
app.py CHANGED
@@ -1,396 +1,349 @@
1
- import gradio as gr
2
- import openai
3
- import os
4
- import subprocess
5
- import threading
6
- import time
7
- import base64
8
- import io
9
- from PIL import Image
10
-
11
- # --- MODIFIED: Use pure-python-adb (ppadb) library ---
12
- # This library connects to the ADB server, avoiding direct USB permission issues.
13
- try:
14
- from ppadb.client import Client as AdbClient
15
- ADB_AVAILABLE = True
16
- except ImportError:
17
- ADB_AVAILABLE = False
18
- print("❌ Critical Error: pure-python-adb library not found. Android integration will not work.")
19
- print(" Please install it with: pip install pure-python-adb")
20
-
21
- """
22
- Nebius AI Studio OpenAI-compatible API integration with Mistral model + Android Integration
23
- (Enhanced with ppadb for robust device connection)
24
- """
25
-
26
- # Initialize OpenAI client with Nebius configuration
27
- def get_openai_client():
28
- api_key = os.getenv("nebiusaistudioapikey")
29
- if not api_key:
30
- raise ValueError("NEBIUSAISTUDIOAPIKEY environment variable is not set. Please configure your API key.")
31
-
32
- return openai.OpenAI(
33
- api_key=api_key,
34
- base_url="https://api.studio.nebius.ai/v1/"
35
- )
36
-
37
- # Initialize client (will be created when needed)
38
- client = None
39
-
40
- def respond(message, history, system_message, max_tokens, temperature, top_p, image_data=None):
41
- global client
42
- max_history = 20
43
- history = history[-max_history:]
44
-
45
- if client is None:
46
- try:
47
- client = get_openai_client()
48
- except ValueError as e:
49
- yield history + [{"role": "assistant", "content": f"Configuration Error: {str(e)}"}]
50
- return
51
-
52
- messages = [{"role": "system", "content": system_message}]
53
- for msg in history:
54
- if isinstance(msg, dict):
55
- messages.append(msg)
56
- elif isinstance(msg, (list, tuple)) and len(msg) == 2:
57
- if msg[0]: messages.append({"role": "user", "content": msg[0]})
58
- if msg[1]: messages.append({"role": "assistant", "content": msg[1]})
59
-
60
- if image_data:
61
- # The API expects the image first in the content list for multimodal input
62
- content = [
63
- {"type": "image_url", "image_url": {"url": image_data}},
64
- {"type": "text", "text": message}
65
- ]
66
- messages.append({"role": "user", "content": content})
67
- else:
68
- messages.append({"role": "user", "content": message})
69
-
70
-
71
- try:
72
- response = client.chat.completions.create(
73
- model="mistralai/Mistral-Small-3.1-24B-Instruct-2503", # Note: Check if this model supports vision
74
- messages=messages,
75
- max_tokens=max_tokens,
76
- temperature=temperature,
77
- top_p=top_p,
78
- stream=True,
79
- )
80
- partial_message = ""
81
- # The user message is already in the history before calling respond
82
- # So we just append the assistant's response
83
- full_history = history + [{"role": "user", "content": message}]
84
- for chunk in response:
85
- if chunk.choices[0].delta.content is not None:
86
- partial_message += chunk.choices[0].delta.content
87
- yield full_history + [{"role": "assistant", "content": partial_message}]
88
- except Exception as e:
89
- yield history + [{"role": "user", "content": message}, {"role": "assistant", "content": f"Error: {str(e)}"}]
90
-
91
- # --- ENHANCED: Android Device Management with ppadb ---
92
- class AndroidDeviceManager:
93
- def __init__(self):
94
- self.adb_client = None
95
- self.connected_devices_list = []
96
- self.current_device_serial = None
97
- self.current_ppadb_device = None # This will hold the ppadb device object
98
- self.latest_screenshot = None
99
- self.latest_screenshot_uri = None
100
- self.automation_status = "Ready"
101
-
102
- if ADB_AVAILABLE:
103
- try:
104
- # Connect to the ADB server on localhost
105
- self.adb_client = AdbClient(host="127.0.0.1", port=5037)
106
- print("βœ… Successfully initialized ADB client.")
107
- except Exception as e:
108
- print(f"❌ Failed to initialize ADB client: {e}")
109
- print(" Ensure the ADB server is running. You can start it with 'adb start-server'")
110
- self.adb_client = None
111
-
112
- def _create_mock_device_for_demo(self):
113
- """Creates a simulated device for demonstration purposes."""
114
- self.current_device_serial = "demo_device_cloud (Simulated)"
115
- self.current_ppadb_device = None
116
- try:
117
- # Create a more realistic demo image
118
- demo_img = Image.new('RGB', (1080, 2340), color='#f0f2f5')
119
- from PIL import ImageDraw, ImageFont
120
- draw = ImageDraw.Draw(demo_img)
121
- # You might need to specify a font file path for ImageFont
122
- try:
123
- font_h1 = ImageFont.truetype("arial.ttf", 60)
124
- font_p = ImageFont.truetype("arial.ttf", 45)
125
- except IOError:
126
- font_h1 = ImageFont.load_default()
127
- font_p = ImageFont.load_default()
128
-
129
- draw.rectangle([0, 0, 1080, 100], fill='#4a90e2') # Status bar
130
- draw.text((40, 150), "Demo App", fill='black', font=font_h1)
131
- draw.rectangle([100, 300, 980, 450], fill='#ffffff', outline='gray', width=3)
132
- draw.text((400, 345), "Login", fill='black', font=font_p)
133
- draw.rectangle([100, 500, 980, 650], fill='#ffffff', outline='gray', width=3)
134
- draw.text((380, 545), "Sign Up", fill='black', font=font_p)
135
-
136
- self.latest_screenshot = self._resize_image(demo_img)
137
- self.latest_screenshot_uri = self._image_to_data_uri(self.latest_screenshot)
138
- return True
139
- except Exception as e:
140
- print(f"Error creating demo device: {e}")
141
- return False
142
-
143
- def get_connected_devices(self):
144
- """Gets device serial numbers from the ADB server."""
145
- demo_option = "demo_device_cloud (Simulated)"
146
- if not self.adb_client:
147
- return [demo_option]
148
- try:
149
- # This ensures the server is running before listing devices
150
- self.adb_client.version()
151
- devices = self.adb_client.devices()
152
- self.connected_devices_list = [d.serial for d in devices]
153
- all_options = self.connected_devices_list + [demo_option]
154
- if not devices:
155
- print("⚠️ No physical ADB devices found. Only demo device is available.")
156
- return all_options
157
- except Exception as e:
158
- print(f"❌ ADB command failed: {e}. Is the ADB server running?")
159
- print(" Try running 'adb start-server' in your terminal.")
160
- return [demo_option]
161
-
162
- def connect_device(self, device_serial):
163
- """Connects to a device by its serial number."""
164
- if not device_serial:
165
- return False, "No device selected."
166
- if "demo_device_cloud" in device_serial:
167
- success = self._create_mock_device_for_demo()
168
- return success, "Connected to Demo Device." if success else "Failed to create Demo Device."
169
-
170
- if not self.adb_client:
171
- return False, "ADB client not available."
172
-
173
- try:
174
- self.current_ppadb_device = self.adb_client.device(device_serial)
175
- if self.current_ppadb_device:
176
- self.current_device_serial = device_serial
177
- print(f"βœ… Successfully connected to {device_serial}")
178
- return True, f"Connected to {device_serial}"
179
- else:
180
- self.current_device_serial = None
181
- self.current_ppadb_device = None
182
- print(f"❌ Failed to connect to device '{device_serial}'. It may have disconnected.")
183
- return False, f"Failed to connect to {device_serial}."
184
- except Exception as e:
185
- print(f"❌ Device connection error for '{device_serial}': {e}")
186
- self.current_device_serial = None
187
- self.current_ppadb_device = None
188
- return False, f"Error connecting to {device_serial}."
189
-
190
- def _resize_image(self, img, max_width=450):
191
- """Resizes a PIL image to a max width, preserving aspect ratio."""
192
- if img.width <= max_width:
193
- return img
194
- ratio = max_width / float(img.width)
195
- height = int(float(img.height) * ratio)
196
- return img.resize((max_width, height), Image.Resampling.LANCZOS)
197
-
198
- def _image_to_data_uri(self, img):
199
- """Converts a PIL image to a base64 data URI."""
200
- buffered = io.BytesIO()
201
- img.save(buffered, format="PNG")
202
- return f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}"
203
-
204
- def capture_screenshot(self):
205
- """Captures a screenshot from the connected device."""
206
- if not self.current_device_serial: return None, "No device connected."
207
- if "demo_device_cloud" in self.current_device_serial:
208
- return self.latest_screenshot, "Captured from Demo Device."
209
- if not self.current_ppadb_device: return None, "Device not properly connected."
210
-
211
- try:
212
- # ppadb has a direct screencap method
213
- result_bytes = self.current_ppadb_device.screencap()
214
- img = Image.open(io.BytesIO(result_bytes))
215
-
216
- img_resized = self._resize_image(img)
217
- self.latest_screenshot = img_resized
218
- self.latest_screenshot_uri = self._image_to_data_uri(img_resized)
219
- return self.latest_screenshot, "Screenshot captured successfully."
220
- except Exception as e:
221
- print(f"Screenshot error: {e}")
222
- return self.latest_screenshot, f"Screenshot error: {e}"
223
-
224
- def get_device_info(self):
225
- if not self.current_device_serial: return "No device connected"
226
- if "demo_device_cloud" in self.current_device_serial: return "Model: Demo Android Device\nAndroid: 14.0"
227
- if not self.current_ppadb_device: return "ADB device not properly connected"
228
- try:
229
- model = self.current_ppadb_device.shell('getprop ro.product.model').strip()
230
- version = self.current_ppadb_device.shell('getprop ro.build.version.release').strip()
231
- # The 'wm size' command is simple with ppadb
232
- resolution = self.current_ppadb_device.shell('wm size').strip().split(': ')[-1]
233
- return f"Model: {model}\nAndroid: {version}\nResolution: {resolution}"
234
- except Exception as e:
235
- return f"Error getting device info: {e}"
236
-
237
- def device_action(self, action_func, *args):
238
- """Wrapper to perform an action and then take a screenshot."""
239
- if self.current_ppadb_device:
240
- action_func(*args)
241
- time.sleep(1.0) # Give UI time to update before screenshot
242
- img, _ = self.capture_screenshot()
243
- return img
244
- return self.latest_screenshot
245
-
246
- def send_tap(self, x, y):
247
- self.current_ppadb_device.shell(f'input tap {int(x)} {int(y)}')
248
-
249
- def send_text(self, text):
250
- # ppadb's input_text is more reliable than shell command
251
- self.current_ppadb_device.input_text(text)
252
-
253
- def press_key(self, key):
254
- key_map = {'back': 4, 'home': 3, 'menu': 82, 'enter': 66}
255
- self.current_ppadb_device.shell(f'input keyevent {key_map.get(key, 0)}')
256
-
257
- # Initialize device manager
258
- device_manager = AndroidDeviceManager()
259
-
260
- # Custom CSS for UI styling
261
- custom_css = """
262
- .gradio-container { font-family: 'Segoe UI', sans-serif; }
263
- .device-info { background: #f8f9fa; border-radius: 8px; padding: 10px; margin: 5px 0; border-left: 4px solid #4CAF50; }
264
- h1 { text-align: center; font-size: 2.5em; margin-bottom: 10px; }
265
- """
266
-
267
- # System prompts for different AI roles
268
- system_prompts = {
269
- "Android App Reviewer": "You are an expert QA Engineer and UX/UI analyst. Your goal is to review the Android application screen provided. Analyze the layout, text, and elements. Provide a concise, actionable report in markdown format. Identify potential bugs, suggest UI/UX improvements, and comment on overall usability. Be specific in your feedback (e.g., 'The 'Login' button contrast is too low,' not 'button looks bad').",
270
- "UX/UI Analyzer": "You are a world-class UX/UI designer. Your task is to analyze the provided mobile app screen. Focus on design principles, user flow, accessibility, and visual hierarchy. Provide constructive feedback with clear justifications.",
271
- "General Assistant": "You are a helpful AI assistant. Provide accurate and thoughtful responses to the user's query about the provided image or text.",
272
- }
273
-
274
- # --- MAIN GRADIO UI ---
275
- def create_android_interface():
276
- with gr.Blocks(css=custom_css, title="Android App Review Agent") as demo:
277
- gr.HTML("<h1>πŸ€– Android App Review Agent</h1>")
278
-
279
- with gr.Row():
280
- # Left Column: Chat Interface
281
- with gr.Column(scale=2):
282
- gr.HTML("<h3>πŸ’¬ AI Assistant</h3>")
283
- chatbot = gr.Chatbot(height=500, show_label=False, show_copy_button=True, type="messages")
284
- with gr.Row():
285
- msg = gr.Textbox(placeholder="Ask about the screen, or give a command...", show_label=False, scale=4)
286
- submit_btn = gr.Button("Send πŸš€", variant="primary", scale=1)
287
- with gr.Row():
288
- clear_btn = gr.Button("πŸ—‘οΈ Clear Chat", variant="secondary")
289
- include_screen = gr.Checkbox(label="Include screen in AI context", value=True)
290
-
291
- # Middle Column: Android Device
292
- with gr.Column(scale=2):
293
- gr.HTML("<h3>πŸ“± Android Device</h3>")
294
- with gr.Row():
295
- refresh_btn = gr.Button("πŸ”„ Refresh Devices", variant="secondary", scale=2)
296
- device_dropdown = gr.Dropdown(choices=[], label="Connected Devices", scale=3, interactive=True)
297
- connect_btn = gr.Button("πŸ”Œ Connect to Selected Device", variant="primary")
298
- device_status = gr.HTML("<div class='device-info'><p><strong>Status:</strong> No device connected</p></div>")
299
- device_screen = gr.Image(height=600, show_label=False, interactive=False)
300
- with gr.Row():
301
- back_btn = gr.Button("⬅️ Back")
302
- home_btn = gr.Button("🏠 Home")
303
- screenshot_btn = gr.Button("πŸ“Έ Capture Screen", variant="primary")
304
-
305
- # Right Column: Controls
306
- with gr.Column(scale=1):
307
- gr.HTML("<h3>βš™οΈ Controls</h3>")
308
- with gr.Accordion("🎭 AI Role", open=True):
309
- prompt_preset = gr.Dropdown(choices=list(system_prompts.keys()), value="Android App Reviewer", label="Preset")
310
- system_message = gr.Textbox(value=system_prompts["Android App Reviewer"], label="Custom System Prompt", lines=5)
311
- with gr.Accordion("πŸŽ›οΈ AI Settings", open=False):
312
- max_tokens = gr.Slider(1, 4096, 2048, label="Max Tokens")
313
- temperature = gr.Slider(0.0, 1.0, 0.5, label="Temperature")
314
- top_p = gr.Slider(0.0, 1.0, 0.95, label="Top-p")
315
- with gr.Accordion("πŸ‘† Device Interaction", open=True):
316
- gr.Markdown("**Tap Coordinates (x, y)**")
317
- with gr.Row():
318
- tap_x = gr.Number(label="X", value=300)
319
- tap_y = gr.Number(label="Y", value=500)
320
- tap_btn = gr.Button("πŸ‘† Tap", variant="primary")
321
- text_input = gr.Textbox(label="Text to Send to Device")
322
- send_text_btn = gr.Button("⌨️ Send Text", variant="primary")
323
-
324
- # --- EVENT HANDLERS ---
325
- def refresh_devices_ui():
326
- devices = device_manager.get_connected_devices()
327
- current_val = device_manager.current_device_serial
328
- # If current device is still connected, keep it selected
329
- if current_val and current_val in devices:
330
- return gr.Dropdown(choices=devices, value=current_val)
331
- # Otherwise, select the first available device or none
332
- return gr.Dropdown(choices=devices, value=devices[0] if devices else None)
333
-
334
- def connect_device_ui(device_id):
335
- success, message = device_manager.connect_device(device_id)
336
- if success:
337
- info = device_manager.get_device_info().replace('\n', '<br>')
338
- status_html = f"<div class='device-info' style='border-left-color: #28a745;'><strong>Status:</strong> βœ… {message}<br>{info}</div>"
339
- screenshot, _ = device_manager.capture_screenshot()
340
- return status_html, screenshot
341
- else:
342
- return f"<div class='device-info' style='border-left-color: #dc3545;'><strong>Status:</strong> ❌ {message} Check console for details.</div>", None
343
-
344
- def chat_with_optional_screen(message, history, system_msg, max_tok, temp, top_p_val, use_screen):
345
- image_data = None
346
- if use_screen and device_manager.current_device_serial and device_manager.latest_screenshot_uri:
347
- image_data = device_manager.latest_screenshot_uri
348
-
349
- # Clear the input box immediately for better UX
350
- yield history + [{"role": "user", "content": message}], ""
351
-
352
- # Start streaming the response
353
- final_response = None
354
- for response in respond(message, history, system_msg, max_tok, temp, top_p_val, image_data):
355
- final_response = response
356
- yield final_response, ""
357
-
358
- def update_screen_and_status(img, status_msg):
359
- return img, gr.update(value=f"<div class='device-info'>{status_msg}</div>") if status_msg else gr.update()
360
-
361
- # Connect UI components to functions
362
- demo.load(refresh_devices_ui, outputs=device_dropdown)
363
- refresh_btn.click(refresh_devices_ui, outputs=device_dropdown)
364
- connect_btn.click(connect_device_ui, inputs=device_dropdown, outputs=[device_status, device_screen])
365
-
366
- prompt_preset.change(lambda preset: system_prompts.get(preset, ""), inputs=prompt_preset, outputs=system_message)
367
-
368
- # Chat submission logic
369
- msg.submit(chat_with_optional_screen, [msg, chatbot, system_message, max_tokens, temperature, top_p, include_screen], [chatbot, msg])
370
- submit_btn.click(chat_with_optional_screen, [msg, chatbot, system_message, max_tokens, temperature, top_p, include_screen], [chatbot, msg])
371
- clear_btn.click(lambda: ([], ""), outputs=[chatbot, msg])
372
-
373
- # Device action logic
374
- screenshot_btn.click(device_manager.capture_screenshot, outputs=[device_screen, device_status])
375
- back_btn.click(lambda: device_manager.device_action(device_manager.press_key, "back"), outputs=[device_screen])
376
- home_btn.click(lambda: device_manager.device_action(device_manager.press_key, "home"), outputs=[device_screen])
377
- tap_btn.click(lambda x, y: device_manager.device_action(device_manager.send_tap, x, y), [tap_x, tap_y], device_screen)
378
- send_text_btn.click(lambda text: device_manager.device_action(device_manager.send_text, text), [text_input], device_screen).then(lambda: "", outputs=text_input)
379
-
380
-
381
- return demo
382
-
383
- if __name__ == "__main__":
384
- print("===== Application Startup =====")
385
- if not os.getenv("nebiusaistudioapikey"):
386
- print("⚠️ Warning: NEBIUSAISTUDIOAPIKEY environment variable not set! API calls will fail.")
387
- else:
388
- print("βœ… Nebius AI Studio API key found.")
389
-
390
- if ADB_AVAILABLE:
391
- print("βœ… pure-python-adb library is available.")
392
-
393
- app = create_android_interface()
394
-
395
- print("\nπŸš€ Starting Gradio interface...")
396
  app.launch()
 
1
+ import gradio as gr
2
+ import openai
3
+ import os
4
+ import io
5
+ import base64
6
+ import asyncio
7
+ from PIL import Image, ImageDraw, ImageFont
8
+
9
+ # --- ADB Library Integration ---
10
+ try:
11
+ from ppadb.client import Client as AdbClient
12
+ ADB_AVAILABLE = True
13
+ except ImportError:
14
+ ADB_AVAILABLE = False
15
+ print("❌ Critical Error: pure-python-adb library not found. Android integration will not work.")
16
+ print(" Please install it with: pip install pure-python-adb")
17
+
18
+ """
19
+ Nebius AI Studio OpenAI-compatible API integration with Mistral model + Android Integration
20
+ (Enhanced with asyncio for robust async operation in Gradio and improved error handling for HF Spaces)
21
+ """
22
+
23
+ # --- OpenAI Client Initialization ---
24
+ def get_openai_client():
25
+ api_key = os.getenv("nebiusaistudioapikey") # Standard practice is all-caps for env vars
26
+ if not api_key:
27
+ raise ValueError("NEBIUSAISTUDIOAPIKEY environment variable not set in your Space secrets.")
28
+ return openai.OpenAI(api_key=api_key, base_url="https://api.studio.nebius.ai/v1/")
29
+
30
+ client = None
31
+
32
+ # --- AI Chat Function ---
33
+ async def respond(message, history, system_message, max_tokens, temperature, top_p, image_data=None):
34
+ global client
35
+ max_history = 20
36
+ history = history[-max_history:]
37
+
38
+ if client is None:
39
+ try:
40
+ client = get_openai_client()
41
+ except ValueError as e:
42
+ yield history + [{"role": "assistant", "content": f"Configuration Error: {str(e)}"}]
43
+ return
44
+
45
+ messages = [{"role": "system", "content": system_message}]
46
+ for role, content in history:
47
+ messages.append({"role": role, "content": content})
48
+
49
+ user_content = [{"type": "text", "text": message}]
50
+ if image_data:
51
+ # Vision models prefer the image first
52
+ user_content.insert(0, {"type": "image_url", "image_url": {"url": image_data}})
53
+
54
+ messages.append({"role": "user", "content": user_content})
55
+
56
+ try:
57
+ response = await asyncio.to_thread(
58
+ client.chat.completions.create,
59
+ model="mistralai/Mistral-Small-3.1-24B-Instruct-2503", # NOTE: Ensure this model supports vision
60
+ messages=messages,
61
+ max_tokens=max_tokens,
62
+ temperature=temperature,
63
+ top_p=top_p,
64
+ stream=True,
65
+ )
66
+
67
+ # The user message is already in the history before calling respond
68
+ # We just need to append the assistant's response part by part
69
+ partial_message = ""
70
+ for chunk in response:
71
+ if chunk.choices[0].delta.content is not None:
72
+ partial_message += chunk.choices[0].delta.content
73
+ yield history + [("user", message), ("assistant", partial_message)]
74
+
75
+ except Exception as e:
76
+ yield history + [("user", message), ("assistant", f"API Error: {str(e)}")]
77
+
78
+
79
+ # --- Android Device Management (Refactored for Async) ---
80
+ class AndroidDeviceManager:
81
+ def __init__(self):
82
+ self.adb_client = None
83
+ self.adb_server_running = False
84
+ self.current_device_serial = None
85
+ self.current_ppadb_device = None
86
+ self.latest_screenshot_uri = None
87
+
88
+ if ADB_AVAILABLE:
89
+ try:
90
+ # Check for ADB server connection on initialization
91
+ self.adb_client = AdbClient(host="127.0.0.1", port=5037)
92
+ self.adb_client.version() # This will raise an exception if server isn't running
93
+ self.adb_server_running = True
94
+ print("βœ… ADB server found and client initialized.")
95
+ except Exception as e:
96
+ self.adb_client = None
97
+ self.adb_server_running = False
98
+ print(f"⚠️ ADB server not found on localhost:5037. Reason: {e}")
99
+ print(" Will operate in 'Simulated Device' mode only. This is expected on Hugging Face Spaces.")
100
+
101
+ def _create_mock_device_for_demo(self):
102
+ """Creates and sets up the simulated device."""
103
+ self.current_device_serial = "demo_device_cloud (Simulated)"
104
+ self.current_ppadb_device = None
105
+ try:
106
+ demo_img = Image.new('RGB', (1080, 2340), color='#f0f2f5')
107
+ draw = ImageDraw.Draw(demo_img)
108
+ # Use a default font to avoid file not found errors on servers
109
+ font_h1 = ImageFont.load_default(size=60)
110
+ font_p = ImageFont.load_default(size=45)
111
+ draw.rectangle([0, 0, 1080, 100], fill='#4a90e2')
112
+ draw.text((40, 150), "Demo App", fill='black', font=font_h1)
113
+ draw.rectangle([100, 300, 980, 450], fill='#ffffff', outline='gray', width=3)
114
+ draw.text((400, 345), "Login", fill='black', font=font_p)
115
+ self.latest_screenshot_uri = self._image_to_data_uri(demo_img)
116
+ return True, "Connected to Simulated Device."
117
+ except Exception as e:
118
+ print(f"Error creating demo device: {e}")
119
+ return False, "Failed to create Simulated Device."
120
+
121
+ def get_connected_devices(self):
122
+ """BLOCKING: Gets device serial numbers. Must be run in a thread."""
123
+ demo_option = "demo_device_cloud (Simulated)"
124
+ if not self.adb_server_running:
125
+ return [demo_option]
126
+ try:
127
+ devices = self.adb_client.devices()
128
+ return [d.serial for d in devices] + [demo_option]
129
+ except Exception as e:
130
+ print(f"❌ ADB command failed: {e}. Falling back to demo mode.")
131
+ self.adb_server_running = False # Mark as not running to avoid retries
132
+ return [demo_option]
133
+
134
+ def connect_device(self, device_serial):
135
+ """BLOCKING: Connects to a device. Must be run in a thread."""
136
+ if not device_serial:
137
+ return False, "No device selected."
138
+ if "demo_device_cloud" in device_serial:
139
+ return self._create_mock_device_for_demo()
140
+
141
+ if not self.adb_server_running:
142
+ return False, "ADB Server not available. Cannot connect to physical devices."
143
+
144
+ try:
145
+ device = self.adb_client.device(device_serial)
146
+ if device:
147
+ self.current_ppadb_device = device
148
+ self.current_device_serial = device_serial
149
+ print(f"βœ… Successfully connected to {device_serial}")
150
+ return True, f"Connected to {device_serial}"
151
+ else:
152
+ raise RuntimeError("Device not found by client.")
153
+ except Exception as e:
154
+ self.current_device_serial = None
155
+ self.current_ppadb_device = None
156
+ return False, f"Failed to connect to {device_serial}: {e}"
157
+
158
+ def _image_to_data_uri(self, img, max_width=800):
159
+ """Resizes a PIL image and converts to a base64 data URI."""
160
+ if img.width > max_width:
161
+ ratio = max_width / float(img.width)
162
+ height = int(float(img.height) * ratio)
163
+ img = img.resize((max_width, height), Image.Resampling.LANCZOS)
164
+
165
+ buffered = io.BytesIO()
166
+ img.save(buffered, format="PNG")
167
+ return f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}"
168
+
169
+ def capture_screenshot(self):
170
+ """BLOCKING: Captures a screenshot. Must be run in a thread."""
171
+ if not self.current_device_serial: return None, "No device connected."
172
+ if "demo_device_cloud" in self.current_device_serial:
173
+ # Re-create demo image to simulate a fresh capture
174
+ self._create_mock_device_for_demo()
175
+ return self.latest_screenshot_uri, "Captured from Simulated Device."
176
+ if not self.current_ppadb_device: return None, "Device not properly connected."
177
+
178
+ try:
179
+ result_bytes = self.current_ppadb_device.screencap()
180
+ img = Image.open(io.BytesIO(result_bytes))
181
+ self.latest_screenshot_uri = self._image_to_data_uri(img)
182
+ return self.latest_screenshot_uri, "Screenshot captured successfully."
183
+ except Exception as e:
184
+ return None, f"Screenshot error: {e}"
185
+
186
+ def get_device_info(self):
187
+ """BLOCKING: Gets device info. Must be run in a thread."""
188
+ if not self.current_device_serial: return "No device connected"
189
+ if "demo_device_cloud" in self.current_device_serial: return "Model: Simulated Android\nOS: 14.0"
190
+ if not self.current_ppadb_device: return "ADB device object not found."
191
+ try:
192
+ model = self.current_ppadb_device.shell('getprop ro.product.model').strip()
193
+ version = self.current_ppadb_device.shell('getprop ro.build.version.release').strip()
194
+ return f"Model: {model}\nOS: {version}"
195
+ except Exception as e:
196
+ return f"Error getting device info: {e}"
197
+
198
+ def perform_action(self, action_name, *args):
199
+ """BLOCKING: Generic action executor. Must be run in a thread."""
200
+ if not self.current_ppadb_device:
201
+ return self.latest_screenshot_uri, "Action failed: No real device connected."
202
+ try:
203
+ action_map = {
204
+ 'tap': lambda x, y: self.current_ppadb_device.shell(f'input tap {int(x)} {int(y)}'),
205
+ 'text': lambda text: self.current_ppadb_device.input_text(text),
206
+ 'key': lambda key: self.current_ppadb_device.shell(f'input keyevent {{"back": 4, "home": 3}.get(key, 0)}')
207
+ }
208
+ action_map[action_name](*args)
209
+ # Give UI time to update before screenshot
210
+ asyncio.run(asyncio.sleep(1.0)) # Use asyncio.sleep in sync context
211
+ return self.capture_screenshot()
212
+ except Exception as e:
213
+ return self.latest_screenshot_uri, f"Action '{action_name}' failed: {e}"
214
+
215
+ # Initialize device manager
216
+ device_manager = AndroidDeviceManager()
217
+
218
+ # --- MAIN GRADIO UI ---
219
+ def create_android_interface():
220
+ with gr.Blocks(css=".gradio-container{font-family:'Segoe UI',sans-serif}", title="Android App Review Agent") as demo:
221
+ gr.HTML("<h1>πŸ€– Android App Review Agent</h1>")
222
+
223
+ with gr.Row():
224
+ with gr.Column(scale=2):
225
+ chatbot = gr.Chatbot(height=550, label="AI Assistant", show_copy_button=True)
226
+ with gr.Row():
227
+ msg = gr.Textbox(placeholder="Ask about the screen...", show_label=False, scale=4)
228
+ submit_btn = gr.Button("Send", variant="primary", scale=1)
229
+ with gr.Row():
230
+ clear_btn = gr.Button("πŸ—‘οΈ Clear Chat", variant="secondary")
231
+ include_screen = gr.Checkbox(label="Include screen in prompt", value=True)
232
+
233
+ with gr.Column(scale=2):
234
+ gr.HTML("<h3>πŸ“± Android Device</h3>")
235
+ with gr.Row():
236
+ refresh_btn = gr.Button("πŸ”„ Refresh Devices", scale=1)
237
+ device_dropdown = gr.Dropdown(label="Connected Devices", scale=2, interactive=True)
238
+ connect_btn = gr.Button("πŸ”Œ Connect", variant="primary")
239
+ device_status = gr.HTML(f"<div style='padding:10px;border-radius:5px;background-color:#f0f0f0;'>Status: App Loaded</div>")
240
+ device_screen = gr.Image(label="Device Screen", height=600, interactive=False)
241
+ with gr.Row():
242
+ back_btn = gr.Button("⬅️ Back")
243
+ home_btn = gr.Button("🏠 Home")
244
+ screenshot_btn = gr.Button("πŸ“Έ Capture", variant="primary")
245
+
246
+ with gr.Column(scale=1):
247
+ gr.HTML("<h3>βš™οΈ Controls</h3>")
248
+ system_prompts = {
249
+ "Android App Reviewer": "You are an expert QA Engineer. Analyze the provided app screen. Report potential bugs, UI/UX improvements, and usability issues in a concise markdown report.",
250
+ "General Assistant": "You are a helpful AI assistant. Provide accurate responses to the user's query.",
251
+ }
252
+ system_message = gr.Textbox(value=system_prompts["Android App Reviewer"], label="System Prompt", lines=8)
253
+ gr.Accordion("πŸŽ›οΈ AI Settings", open=False)
254
+ max_tokens = gr.Slider(1, 4096, 2048, label="Max Tokens")
255
+ temperature = gr.Slider(0.0, 1.0, 0.5, label="Temperature")
256
+ top_p = gr.Slider(0.0, 1.0, 0.95, label="Top-p")
257
+ with gr.Accordion("πŸ‘† Device Interaction", open=True):
258
+ with gr.Row():
259
+ tap_x = gr.Number(label="X", value=540)
260
+ tap_y = gr.Number(label="Y", value=1170)
261
+ tap_btn = gr.Button("πŸ‘† Tap")
262
+ text_input = gr.Textbox(label="Text to Send")
263
+ send_text_btn = gr.Button("⌨️ Send Text")
264
+
265
+ # --- ASYNC EVENT HANDLERS ---
266
+ async def async_refresh_devices():
267
+ devices = await asyncio.to_thread(device_manager.get_connected_devices)
268
+ current_val = device_manager.current_device_serial
269
+ new_value = current_val if current_val in devices else devices[0] if devices else None
270
+ return gr.Dropdown(choices=devices, value=new_value)
271
+
272
+ async def async_connect_device(device_id):
273
+ if not device_id:
274
+ return "<div>Status: Please select a device.</div>", None
275
+
276
+ success, message = await asyncio.to_thread(device_manager.connect_device, device_id)
277
+ color = "#28a745" if success else "#dc3545"
278
+ status_html = f"<div style='padding:10px;border-radius:5px;background-color:{color};color:white;'><strong>Status:</strong> {message}</div>"
279
+
280
+ if success:
281
+ info = await asyncio.to_thread(device_manager.get_device_info)
282
+ status_html += f"<div style='padding:10px;margin-top:5px;border-radius:5px;background-color:#f0f0f0;'>{info.replace(chr(10), '<br>')}</div>"
283
+ screen_uri, _ = await asyncio.to_thread(device_manager.capture_screenshot)
284
+ return status_html, screen_uri
285
+ else:
286
+ return status_html, None
287
+
288
+ async def async_chat_stream(message, history, sys_msg, max_tok, temp, top_p_val, use_screen):
289
+ if not message:
290
+ return history, ""
291
+
292
+ # Immediately update UI
293
+ yield history + [("user", message)], ""
294
+
295
+ image_data = device_manager.latest_screenshot_uri if use_screen else None
296
+
297
+ # Stream the response
298
+ async for response in respond(message, history, sys_msg, max_tok, temp, top_p_val, image_data):
299
+ yield response, ""
300
+
301
+ async def async_capture_and_update():
302
+ screen_uri, status_msg = await asyncio.to_thread(device_manager.capture_screenshot)
303
+ color = "#28a745" if "success" in status_msg.lower() else "#ffc107"
304
+ status_html = f"<div style='padding:10px;border-radius:5px;background-color:{color};color:white;'>{status_msg}</div>"
305
+ return screen_uri, status_html
306
+
307
+ async def async_device_action(action, *params):
308
+ screen_uri, status_msg = await asyncio.to_thread(device_manager.perform_action, action, *params)
309
+ status_html = f"<div style='padding:10px;border-radius:5px;background-color:#007bff;color:white;'>{status_msg}</div>"
310
+ return screen_uri, status_html
311
+
312
+ # --- UI Event Wiring ---
313
+ async def initial_load():
314
+ """Runs when the app first loads."""
315
+ dropdown_update = await async_refresh_devices()
316
+ # If no ADB server, automatically connect to the demo device for a better UX
317
+ if not device_manager.adb_server_running:
318
+ status_update, screen_update = await async_connect_device("demo_device_cloud (Simulated)")
319
+ return dropdown_update, status_update, screen_update
320
+ return dropdown_update, gr.HTML(), gr.Image()
321
+
322
+ demo.load(initial_load, outputs=[device_dropdown, device_status, device_screen])
323
+
324
+ refresh_btn.click(async_refresh_devices, outputs=device_dropdown)
325
+ connect_btn.click(async_connect_device, inputs=device_dropdown, outputs=[device_status, device_screen])
326
+
327
+ msg.submit(async_chat_stream, [msg, chatbot, system_message, max_tokens, temperature, top_p, include_screen], [chatbot, msg])
328
+ submit_btn.click(async_chat_stream, [msg, chatbot, system_message, max_tokens, temperature, top_p, include_screen], [chatbot, msg])
329
+ clear_btn.click(lambda: ([], ""), outputs=[chatbot, msg])
330
+
331
+ screenshot_btn.click(async_capture_and_update, outputs=[device_screen, device_status])
332
+ back_btn.click(lambda: async_device_action("key", "back"), outputs=[device_screen, device_status])
333
+ home_btn.click(lambda: async_device_action("key", "home"), outputs=[device_screen, device_status])
334
+ tap_btn.click(lambda x, y: async_device_action("tap", x, y), [tap_x, tap_y], [device_screen, device_status])
335
+ send_text_btn.click(lambda t: async_device_action("text", t), [text_input], [device_screen, device_status]).then(lambda: "", outputs=text_input)
336
+
337
+ return demo
338
+
339
+ if __name__ == "__main__":
340
+ print("===== Application Startup =====")
341
+ if not os.getenv("nebiusaistudioapikey"):
342
+ print("⚠️ Warning: NEBIUSAISTUDIOAPIKEY environment variable not set! API calls will fail.")
343
+ else:
344
+ print("βœ… Nebius AI Studio API key found.")
345
+
346
+ app = create_android_interface()
347
+
348
+ print("\nπŸš€ Starting Gradio interface...")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
349
  app.launch()