agh123 commited on
Commit
7113dc0
·
1 Parent(s): 9c27319

refactor the app to use firestore

Browse files
.gitignore CHANGED
@@ -38,4 +38,5 @@ ENV/
38
  .streamlit/
39
 
40
  # Logs
41
- *.log
 
 
38
  .streamlit/
39
 
40
  # Logs
41
+ *.log
42
+ pocketpal-ai-6e230-6fe017a6fd65.json
main.py CHANGED
@@ -7,7 +7,7 @@ from src.components.visualizations import (
7
  render_performance_plots,
8
  render_leaderboard_table,
9
  )
10
- from src.services.api import fetch_leaderboard_data
11
 
12
  # Configure the page
13
  st.set_page_config(
 
7
  render_performance_plots,
8
  render_leaderboard_table,
9
  )
10
+ from src.services.firebase import fetch_leaderboard_data
11
 
12
  # Configure the page
13
  st.set_page_config(
requirements.txt CHANGED
@@ -4,4 +4,5 @@ python-dotenv>=1.0.0
4
  pandas>=2.1.3
5
  plotly>=5.18.0
6
  httpx>=0.25.1
7
- pydantic-settings>=2.0.3
 
 
4
  pandas>=2.1.3
5
  plotly>=5.18.0
6
  httpx>=0.25.1
7
+ pydantic-settings>=2.0.3
8
+ firebase-admin
src/app.py CHANGED
@@ -7,7 +7,7 @@ async def fetch_and_filter_data(
7
  benchmark_label: Optional[str] = None
8
  ) -> pd.DataFrame:
9
  """Fetch and filter data based on parameters"""
10
- from .services.api import fetch_leaderboard_data
11
 
12
  return await fetch_leaderboard_data(
13
  model_name=model_name,
 
7
  benchmark_label: Optional[str] = None
8
  ) -> pd.DataFrame:
9
  """Fetch and filter data based on parameters"""
10
+ from .services.firebase import fetch_leaderboard_data
11
 
12
  return await fetch_leaderboard_data(
13
  model_name=model_name,
src/core/config.py CHANGED
@@ -2,8 +2,7 @@ from pydantic_settings import BaseSettings
2
  from functools import lru_cache
3
 
4
  class Settings(BaseSettings):
5
- API_URL: str = "https://a-ghorbani-ai-phone-benchmark-api.hf.space"
6
- HF_TOKEN: str
7
 
8
  class Config:
9
  case_sensitive = True
 
2
  from functools import lru_cache
3
 
4
  class Settings(BaseSettings):
5
+ FIRESTORE_COLLECTION: str = "benchmarks"
 
6
 
7
  class Config:
8
  case_sensitive = True
src/services/api.py DELETED
@@ -1,38 +0,0 @@
1
- import httpx
2
- import pandas as pd
3
- from typing import Optional, Dict
4
- import streamlit as st
5
- from src.core.config import settings
6
-
7
- async def fetch_leaderboard_data(
8
- model_name: Optional[str] = None,
9
- benchmark_label: Optional[str] = None
10
- ) -> pd.DataFrame:
11
- """Fetch and process leaderboard data"""
12
- params = {}
13
- if model_name and model_name != "All":
14
- params["model_name"] = model_name
15
- if benchmark_label and benchmark_label != "All":
16
- params["benchmark_label"] = benchmark_label
17
-
18
- headers = {
19
- "Authorization": f"Bearer {settings.HF_TOKEN}",
20
- "Accept": "application/json"
21
- }
22
-
23
- try:
24
- async with httpx.AsyncClient() as client:
25
- response = await client.get(
26
- f"{settings.API_URL}/api/v1/leaderboard",
27
- params=params,
28
- headers=headers,
29
- follow_redirects=True
30
- )
31
- response.raise_for_status()
32
- data = response.json()
33
- return pd.DataFrame(data)
34
- except Exception as e:
35
- st.error(f"Error fetching data: {str(e)}")
36
- if hasattr(e, 'response'):
37
- st.error(f"Response: {e.response.text}")
38
- return pd.DataFrame()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/services/firebase.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import firebase_admin
2
+ from firebase_admin import credentials, firestore
3
+ from typing import List, Dict, Optional
4
+ import pandas as pd
5
+ import streamlit as st
6
+
7
+ def initialize_firebase():
8
+ """Initialize Firebase with credentials"""
9
+ try:
10
+ firebase_admin.get_app()
11
+ except ValueError:
12
+ cred = credentials.Certificate('pocketpal-ai-6e230-6fe017a6fd65.json')
13
+ firebase_admin.initialize_app(cred)
14
+ return firestore.client()
15
+
16
+ db = initialize_firebase()
17
+
18
+ def normalize_device_id(device_info: dict) -> str:
19
+ """Normalize device identifier for aggregation"""
20
+ emulator = "/Emulator" if device_info["isEmulator"] else ""
21
+ if device_info["systemName"].lower() == "ios":
22
+ return f"iOS/{device_info['model']}{emulator}"
23
+
24
+ memory_tier = f"{device_info['totalMemory'] // (1024**3)}GB"
25
+ return f"{device_info['brand']}/{device_info['model']}/{memory_tier}{emulator}"
26
+
27
+ def format_params_in_b(params: int) -> float:
28
+ """Format number of parameters in billions"""
29
+ b_value = params / 1e9
30
+ if b_value >= 10:
31
+ return round(b_value, 1)
32
+ elif b_value >= 1:
33
+ return round(b_value, 2)
34
+ else:
35
+ return round(b_value, 3)
36
+
37
+ def format_leaderboard_data(submissions: List[dict]) -> pd.DataFrame:
38
+ """Format submissions for leaderboard display"""
39
+ formatted_data = []
40
+
41
+ for sub in submissions:
42
+ try:
43
+ benchmark_result = sub.get('benchmarkResult', {})
44
+ device_info = sub.get('deviceInfo', {})
45
+
46
+ if not benchmark_result or not device_info:
47
+ continue
48
+
49
+ formatted_data.append({
50
+ "Device": f"{device_info.get('model', 'Unknown')} [Emulator]" if device_info.get('isEmulator') else device_info.get('model', 'Unknown'),
51
+ "Platform": device_info.get('systemName', 'Unknown'),
52
+ "Benchmark": f"{benchmark_result.get('config', {}).get('label', 'Unknown')} (pp: {benchmark_result.get('config', {}).get('pp', 'N/A')}, tg: {benchmark_result.get('config', {}).get('tg', 'N/A')})",
53
+ "Model": benchmark_result.get('modelName', 'Unknown'),
54
+ "Model Size": format_params_in_b(benchmark_result.get('modelNParams', 0)),
55
+ "Prompt Processing": round(benchmark_result.get('ppAvg', 0), 2),
56
+ "Token Generation": round(benchmark_result.get('tgAvg', 0), 2),
57
+ "Memory Usage (%)": benchmark_result.get('peakMemoryUsage', {}).get('percentage'),
58
+ "Memory Usage (GB)": round(benchmark_result.get('peakMemoryUsage', {}).get('used', 0) / (1024**3), 2) if benchmark_result.get('peakMemoryUsage', {}).get('used') else None,
59
+ "Total Memory (GB)": round(device_info.get('totalMemory', 0) / (1024**3), 2),
60
+ "CPU Cores": device_info.get('cpuDetails', {}).get('cores', 'Unknown'),
61
+ "Normalized Device ID": normalize_device_id(device_info),
62
+ "Timestamp": benchmark_result.get('timestamp', 'Unknown'),
63
+ "Model ID": benchmark_result.get('modelId', 'Unknown'),
64
+ "OID": benchmark_result.get('oid'),
65
+ })
66
+ except Exception as e:
67
+ st.warning(f"Error processing submission: {str(e)}")
68
+ continue
69
+
70
+ return pd.DataFrame(formatted_data)
71
+
72
+ async def fetch_leaderboard_data(
73
+ model_name: Optional[str] = None,
74
+ benchmark_label: Optional[str] = None
75
+ ) -> pd.DataFrame:
76
+ """Fetch and process leaderboard data from Firestore"""
77
+ try:
78
+ # Navigate to the correct collection path: benchmarks/v1/submissions
79
+ submissions_ref = db.collection('benchmarks').document('v1').collection('submissions')
80
+
81
+ # Get all documents
82
+ docs = submissions_ref.stream()
83
+ all_docs = list(docs)
84
+
85
+ if len(all_docs) == 0:
86
+ return pd.DataFrame()
87
+
88
+ # Process documents and filter in memory
89
+ submissions = []
90
+
91
+ for doc in all_docs:
92
+ data = doc.to_dict()
93
+
94
+ if not data or 'benchmarkResult' not in data:
95
+ continue
96
+
97
+ benchmark_result = data['benchmarkResult']
98
+
99
+ # Apply filters
100
+ if model_name and model_name != "All" and benchmark_result.get('modelName') != model_name:
101
+ continue
102
+ if benchmark_label and benchmark_label != "All" and benchmark_result.get('config', {}).get('label') != benchmark_label:
103
+ continue
104
+
105
+ submissions.append(data)
106
+
107
+ return format_leaderboard_data(submissions)
108
+
109
+ except Exception as e:
110
+ st.error(f"Error fetching data from Firestore: {str(e)}")
111
+ return pd.DataFrame()