Spaces:
Runtime error
Runtime error
CognitiveScience
commited on
Commit
•
2b49fc4
1
Parent(s):
958c393
Update app.py
Browse files
app.py
CHANGED
@@ -9,10 +9,10 @@ import pandas as pd
|
|
9 |
import math
|
10 |
import json
|
11 |
|
12 |
-
|
13 |
-
|
14 |
#import pandas as pd
|
15 |
-
|
16 |
import os
|
17 |
import datetime
|
18 |
from apscheduler.schedulers.background import BackgroundScheduler
|
@@ -21,30 +21,27 @@ import random
|
|
21 |
import time
|
22 |
#import requests
|
23 |
|
24 |
-
|
25 |
-
#repo = huggingface_hub.HfRepository(repo_id="lysandre/test-model", token=token)
|
26 |
|
27 |
-
# Clone the repository to a local directory
|
28 |
-
#repo.clone_from_hub()
|
29 |
#hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./reviews.csv")
|
30 |
|
31 |
-
|
32 |
from datasets import load_dataset
|
33 |
|
34 |
#dataset = load_dataset("csv", data_files="./data.csv")
|
35 |
|
36 |
|
37 |
-
|
38 |
|
39 |
-
|
40 |
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
|
49 |
#TOKEN2 = HF_TOKEN
|
50 |
|
@@ -56,47 +53,39 @@ from datasets import load_dataset
|
|
56 |
|
57 |
# Create table if it doesn't already exist
|
58 |
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
def get_latest_reviews(
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
reviews = df.DataFrame(reviews, columns=["id", "date_created", "name", "view", "duration"])
|
82 |
-
|
83 |
-
except: # sqlite3.OperationalError:
|
84 |
-
df=pd.DataFrame()
|
85 |
-
print ("db ...")
|
86 |
-
#reviews = db.execute("SELECT * FROM reviews ORDER BY id DESC limit 100").fetchall()
|
87 |
-
#total_reviews = db.execute("Select COUNT(id) from reviews").fetchone()[0]
|
88 |
-
total_reviews=reviews.count()[0]
|
89 |
return reviews, total_reviews
|
90 |
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
|
97 |
def ccogsphere(name: str, rate: int, celsci: str):
|
98 |
-
|
99 |
-
|
100 |
|
101 |
#try:
|
102 |
celsci2=celsci.split()
|
@@ -110,13 +99,11 @@ def ccogsphere(name: str, rate: int, celsci: str):
|
|
110 |
duration = str(row["duration"])
|
111 |
print(view, duration)
|
112 |
#celsci=celsci+celsci2
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
reviews=
|
117 |
-
|
118 |
-
#reviews, total_reviews = get_latest_reviews(db)
|
119 |
-
#db.close()
|
120 |
r = requests.post(url='https://ccml-persistent-data2.hf.space/api/predict/', json={"data": [celsci + " ", celsci2]})
|
121 |
|
122 |
return reviews, total_reviews
|
@@ -143,37 +130,33 @@ def run_ecs(inp):
|
|
143 |
try:
|
144 |
result=ecf(inp)
|
145 |
df=pd.DataFrame.from_dict(result["videos"])
|
146 |
-
except
|
147 |
-
|
148 |
-
print ("db ...")
|
149 |
|
150 |
-
|
151 |
|
152 |
-
|
153 |
#cursor = db.cursor()
|
154 |
#cursor.execute("INSERT INTO reviews2(title, link, thumbnail,channel, description, views, uploaded, duration, durationString) VALUES(?,?,?,?,?,?,?,?,?)", [title, link, thumbnail,channel, description, views, uploaded, duration, durationString])
|
155 |
-
|
156 |
|
157 |
#db.commit()
|
158 |
-
|
159 |
-
|
160 |
#print ("print000", total_reviews2,reviews2)
|
161 |
-
reviews2=df
|
162 |
-
total_reviews2=reviews2.count()[0]
|
163 |
-
|
164 |
return reviews2, total_reviews2
|
165 |
|
166 |
|
167 |
-
|
168 |
-
|
169 |
-
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
|
178 |
css="footer {visibility: hidden}"
|
179 |
# Applying style to highlight the maximum value in each row
|
@@ -191,14 +174,61 @@ with gr.Blocks(css=css) as demo:
|
|
191 |
#run_actr()
|
192 |
submit = gr.Button(value=".")
|
193 |
submit.click(ccogsphere, [name, rate, celsci], [data, count])
|
194 |
-
demo.load(
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
|
199 |
-
|
200 |
#if name=="abc":
|
201 |
#run_code()
|
202 |
-
|
203 |
#return "Hello " + name + "!"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
204 |
demo.launch()
|
|
|
9 |
import math
|
10 |
import json
|
11 |
|
12 |
+
import sqlite3
|
13 |
+
import huggingface_hub
|
14 |
#import pandas as pd
|
15 |
+
import shutil
|
16 |
import os
|
17 |
import datetime
|
18 |
from apscheduler.schedulers.background import BackgroundScheduler
|
|
|
21 |
import time
|
22 |
#import requests
|
23 |
|
24 |
+
from huggingface_hub import hf_hub_download
|
|
|
25 |
|
|
|
|
|
26 |
#hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./reviews.csv")
|
27 |
|
28 |
+
from huggingface_hub import login
|
29 |
from datasets import load_dataset
|
30 |
|
31 |
#dataset = load_dataset("csv", data_files="./data.csv")
|
32 |
|
33 |
|
34 |
+
DB_FILE = "./reviewsitr.db"
|
35 |
|
36 |
+
TOKEN = os.environ.get('HF_KEY')
|
37 |
|
38 |
+
repo = huggingface_hub.Repository(
|
39 |
+
local_dir="data",
|
40 |
+
repo_type="dataset",
|
41 |
+
clone_from="CognitiveScience/csdhdata",
|
42 |
+
use_auth_token=TOKEN
|
43 |
+
)
|
44 |
+
repo.git_pull()
|
45 |
|
46 |
#TOKEN2 = HF_TOKEN
|
47 |
|
|
|
53 |
|
54 |
# Create table if it doesn't already exist
|
55 |
|
56 |
+
db = sqlite3.connect(DB_FILE)
|
57 |
+
try:
|
58 |
+
db.execute("SELECT * FROM reviews").fetchall()
|
59 |
+
#db.execute("SELECT * FROM reviews2").fetchall()
|
60 |
+
|
61 |
+
db.close()
|
62 |
+
except sqlite3.OperationalError:
|
63 |
+
db.execute(
|
64 |
+
'''
|
65 |
+
CREATE TABLE reviews (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
|
66 |
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
67 |
+
name TEXT, view TEXT, duration TEXT)
|
68 |
+
''')
|
69 |
+
db.commit()
|
70 |
+
db.close()
|
71 |
+
|
72 |
+
db = sqlite3.connect(DB_FILE)
|
73 |
+
|
74 |
+
def get_latest_reviews(db: sqlite3.Connection):
|
75 |
+
reviews = db.execute("SELECT * FROM reviews ORDER BY id DESC limit 100").fetchall()
|
76 |
+
total_reviews = db.execute("Select COUNT(id) from reviews").fetchone()[0]
|
77 |
+
reviews = pd.DataFrame(reviews, columns=["id", "date_created", "name", "view", "duration"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
78 |
return reviews, total_reviews
|
79 |
|
80 |
+
def get_latest_reviews2(db: sqlite3.Connection):
|
81 |
+
reviews2 = db.execute("SELECT * FROM reviews2 ORDER BY id DESC limit 100").fetchall()
|
82 |
+
total_reviews2 = db.execute("Select COUNT(id) from reviews2").fetchone()[0]
|
83 |
+
reviews2 = pd.DataFrame(reviews2, columns=["id","title", "link","channel", "description", "views", "uploaded", "duration", "durationString"])
|
84 |
+
return reviews2, total_reviews2
|
85 |
|
86 |
def ccogsphere(name: str, rate: int, celsci: str):
|
87 |
+
db = sqlite3.connect(DB_FILE)
|
88 |
+
cursor = db.cursor()
|
89 |
|
90 |
#try:
|
91 |
celsci2=celsci.split()
|
|
|
99 |
duration = str(row["duration"])
|
100 |
print(view, duration)
|
101 |
#celsci=celsci+celsci2
|
102 |
+
cursor.execute("INSERT INTO reviews(name, view, duration) VALUES(?,?,?)", [celsci+str(index+1), view, duration])
|
103 |
+
db.commit()
|
104 |
+
|
105 |
+
reviews, total_reviews = get_latest_reviews(db)
|
106 |
+
db.close()
|
|
|
|
|
107 |
r = requests.post(url='https://ccml-persistent-data2.hf.space/api/predict/', json={"data": [celsci + " ", celsci2]})
|
108 |
|
109 |
return reviews, total_reviews
|
|
|
130 |
try:
|
131 |
result=ecf(inp)
|
132 |
df=pd.DataFrame.from_dict(result["videos"])
|
133 |
+
except sqlite3.OperationalError:
|
134 |
+
print ("db error")
|
|
|
135 |
|
136 |
+
df=df.drop(df.columns[4], axis=1)
|
137 |
|
138 |
+
db = sqlite3.connect(DB_FILE)
|
139 |
#cursor = db.cursor()
|
140 |
#cursor.execute("INSERT INTO reviews2(title, link, thumbnail,channel, description, views, uploaded, duration, durationString) VALUES(?,?,?,?,?,?,?,?,?)", [title, link, thumbnail,channel, description, views, uploaded, duration, durationString])
|
141 |
+
df.to_sql('reviews2', db, if_exists='replace', index=False)
|
142 |
|
143 |
#db.commit()
|
144 |
+
reviews2, total_reviews2 = get_latest_reviews(db)
|
145 |
+
db.close()
|
146 |
#print ("print000", total_reviews2,reviews2)
|
|
|
|
|
|
|
147 |
return reviews2, total_reviews2
|
148 |
|
149 |
|
150 |
+
def load_data():
|
151 |
+
db = sqlite3.connect(DB_FILE)
|
152 |
+
reviews, total_reviews = get_latest_reviews(db)
|
153 |
+
db.close()
|
154 |
+
return reviews, total_reviews
|
155 |
+
def load_data2():
|
156 |
+
db = sqlite3.connect(DB_FILE)
|
157 |
+
reviews2, total_reviews2 = get_latest_reviews2(db)
|
158 |
+
db.close()
|
159 |
+
return reviews2, total_reviews2
|
160 |
|
161 |
css="footer {visibility: hidden}"
|
162 |
# Applying style to highlight the maximum value in each row
|
|
|
174 |
#run_actr()
|
175 |
submit = gr.Button(value=".")
|
176 |
submit.click(ccogsphere, [name, rate, celsci], [data, count])
|
177 |
+
demo.load(load_data, None, [data, count])
|
178 |
+
@name.change(inputs=name, outputs=celsci,_js="window.location.reload()")
|
179 |
+
@rate.change(inputs=rate, outputs=name,_js="window.location.reload()")
|
180 |
+
@celsci.change(inputs=celsci, outputs=rate,_js="window.location.reload()")
|
181 |
|
182 |
+
def secwork(name):
|
183 |
#if name=="abc":
|
184 |
#run_code()
|
185 |
+
load_data()
|
186 |
#return "Hello " + name + "!"
|
187 |
+
def backup_db():
|
188 |
+
shutil.copyfile(DB_FILE, "./reviews.db")
|
189 |
+
db = sqlite3.connect(DB_FILE)
|
190 |
+
reviews = db.execute("SELECT * FROM reviews").fetchall()
|
191 |
+
pd.DataFrame(reviews).to_csv("./reviews.csv", index=False)
|
192 |
+
print("updating db")
|
193 |
+
repo.push_to_hub(blocking=False, commit_message=f"Updating data at {datetime.datetime.now()}")
|
194 |
+
|
195 |
+
def backup_db_csv():
|
196 |
+
shutil.copyfile(DB_FILE, "./reviews2.db")
|
197 |
+
db = sqlite3.connect(DB_FILE)
|
198 |
+
reviews = db.execute("SELECT * FROM reviews").fetchall()
|
199 |
+
pd.DataFrame(reviews).to_csv("./reviews2.csv", index=False)
|
200 |
+
print("updating db csv")
|
201 |
+
dataset = load_dataset("csv", data_files="./reviews2.csv")
|
202 |
+
repo.push_to_hub("CognitiveScience/csdhdata", blocking=False) #, commit_message=f"Updating data-csv at {datetime.datetime.now()}")
|
203 |
+
#path1=hf_hub_url()
|
204 |
+
#print (path1)
|
205 |
+
#hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./*.csv")
|
206 |
+
#hf_hub_download(repo_id="CognitiveScience/csdhdata", filename="./*.db")
|
207 |
+
#hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./*.md")
|
208 |
+
#hf_hub_download(repo_id="CognitiveScience/csdhdata", filename="./*.md")
|
209 |
+
|
210 |
+
|
211 |
+
#def load_data2():
|
212 |
+
# db = sqlite3.connect(DB_FILE)
|
213 |
+
# reviews, total_reviews = get_latest_reviews(db)
|
214 |
+
# #db.close()
|
215 |
+
# demo.load(load_data,None, [reviews, total_reviews])
|
216 |
+
# #return reviews, total_reviews
|
217 |
+
|
218 |
+
#scheduler1 = BackgroundScheduler()
|
219 |
+
#scheduler1.add_job(func=run_actr, trigger="interval", seconds=10)
|
220 |
+
#scheduler1.start()
|
221 |
+
|
222 |
+
scheduler1 = BackgroundScheduler()
|
223 |
+
scheduler1.add_job(func=load_data, trigger="interval", seconds=5)
|
224 |
+
scheduler1.start()
|
225 |
+
|
226 |
+
scheduler2 = BackgroundScheduler()
|
227 |
+
scheduler2.add_job(func=backup_db, trigger="interval", seconds=3633)
|
228 |
+
scheduler2.start()
|
229 |
+
|
230 |
+
scheduler3 = BackgroundScheduler()
|
231 |
+
scheduler3.add_job(func=backup_db_csv, trigger="interval", seconds=3666)
|
232 |
+
scheduler3.start()
|
233 |
+
|
234 |
demo.launch()
|