CognitiveScience commited on
Commit
2b49fc4
1 Parent(s): 958c393

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +118 -88
app.py CHANGED
@@ -9,10 +9,10 @@ import pandas as pd
9
  import math
10
  import json
11
 
12
- #import sqlite3
13
- #import huggingface_hub
14
  #import pandas as pd
15
- #import shutil
16
  import os
17
  import datetime
18
  from apscheduler.schedulers.background import BackgroundScheduler
@@ -21,30 +21,27 @@ import random
21
  import time
22
  #import requests
23
 
24
- #from huggingface_hub import hf_hub_download
25
- #repo = huggingface_hub.HfRepository(repo_id="lysandre/test-model", token=token)
26
 
27
- # Clone the repository to a local directory
28
- #repo.clone_from_hub()
29
  #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./reviews.csv")
30
 
31
- #from huggingface_hub import login
32
  from datasets import load_dataset
33
 
34
  #dataset = load_dataset("csv", data_files="./data.csv")
35
 
36
 
37
- #DB_FILE = "./reviewsitr.db"
38
 
39
- #TOKEN = os.environ.get('HF_KEY')
40
 
41
- #repo = huggingface_hub.Repository(
42
- # local_dir="data",
43
- # repo_type="dataset",
44
- # clone_from="CognitiveScience/csdhdata",
45
- # use_auth_token=TOKEN
46
- #)
47
- #repo.git_pull()
48
 
49
  #TOKEN2 = HF_TOKEN
50
 
@@ -56,47 +53,39 @@ from datasets import load_dataset
56
 
57
  # Create table if it doesn't already exist
58
 
59
- #db = sqlite3.connect(DB_FILE)
60
- #try:
61
- # db.execute("SELECT * FROM reviews").fetchall()
62
- # #db.execute("SELECT * FROM reviews2").fetchall()
63
- #
64
- # db.close()
65
- #except sqlite3.OperationalError:
66
- # db.execute(
67
- # '''
68
- # CREATE TABLE reviews (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
69
- # created_at TIMESTAMP DEFAULT (datetime('now', 'localtime', '+3 hours')) NOT NULL,
70
- # name TEXT, view TEXT, duration TEXT)
71
- # ''')
72
- # db.commit()
73
- # db.close()
74
-
75
- # db = sqlite3.connect(DB_FILE)
76
- # #created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
77
- def get_latest_reviews(celsci2): #db: sqlite3.Connection):
78
- try:
79
- celsci2=ecf(celsci2)
80
- df=pd.DataFrame.from_dict(celsci2["videos"])
81
- reviews = df.DataFrame(reviews, columns=["id", "date_created", "name", "view", "duration"])
82
-
83
- except: # sqlite3.OperationalError:
84
- df=pd.DataFrame()
85
- print ("db ...")
86
- #reviews = db.execute("SELECT * FROM reviews ORDER BY id DESC limit 100").fetchall()
87
- #total_reviews = db.execute("Select COUNT(id) from reviews").fetchone()[0]
88
- total_reviews=reviews.count()[0]
89
  return reviews, total_reviews
90
 
91
- #def get_latest_reviews2(db: sqlite3.Connection):
92
- # reviews2 = db.execute("SELECT * FROM reviews2 ORDER BY id DESC limit 100").fetchall()
93
- # total_reviews2 = db.execute("Select COUNT(id) from reviews2").fetchone()[0]
94
- # reviews2 = pd.DataFrame(reviews2, columns=["id","title", "link","channel", "description", "views", "uploaded", "duration", "durationString"])
95
- # return reviews2, total_reviews2
96
 
97
  def ccogsphere(name: str, rate: int, celsci: str):
98
- #db = sqlite3.connect(DB_FILE)
99
- #cursor = db.cursor()
100
 
101
  #try:
102
  celsci2=celsci.split()
@@ -110,13 +99,11 @@ def ccogsphere(name: str, rate: int, celsci: str):
110
  duration = str(row["duration"])
111
  print(view, duration)
112
  #celsci=celsci+celsci2
113
-
114
- #cursor.execute("INSERT INTO reviews(name, view, duration) VALUES(?,?,?)", [celsci+" Video #"+str(index+1), view, duration])
115
- #db.commit()
116
- reviews=df
117
- total_reviews=reviews.count()[0]
118
- #reviews, total_reviews = get_latest_reviews(db)
119
- #db.close()
120
  r = requests.post(url='https://ccml-persistent-data2.hf.space/api/predict/', json={"data": [celsci + " ", celsci2]})
121
 
122
  return reviews, total_reviews
@@ -143,37 +130,33 @@ def run_ecs(inp):
143
  try:
144
  result=ecf(inp)
145
  df=pd.DataFrame.from_dict(result["videos"])
146
- except: # sqlite3.OperationalError:
147
- df=pd.DataFrame()
148
- print ("db ...")
149
 
150
- #df=df.drop(df.columns[4], axis=1)
151
 
152
- #db = sqlite3.connect(DB_FILE)
153
  #cursor = db.cursor()
154
  #cursor.execute("INSERT INTO reviews2(title, link, thumbnail,channel, description, views, uploaded, duration, durationString) VALUES(?,?,?,?,?,?,?,?,?)", [title, link, thumbnail,channel, description, views, uploaded, duration, durationString])
155
- #df.to_sql('reviews2', db, if_exists='replace', index=False)
156
 
157
  #db.commit()
158
- #reviews2, total_reviews2 = get_latest_reviews(db)
159
- #db.close()
160
  #print ("print000", total_reviews2,reviews2)
161
- reviews2=df
162
- total_reviews2=reviews2.count()[0]
163
-
164
  return reviews2, total_reviews2
165
 
166
 
167
- #def load_data():
168
- # #db = sqlite3.connect(DB_FILE)
169
- # reviews, total_reviews = get_latest_reviews()
170
- # #db.close()
171
- # return reviews, total_reviews
172
- #def load_data2():
173
- # db = sqlite3.connect(DB_FILE)
174
- # reviews2, total_reviews2 = get_latest_reviews2(db)
175
- # db.close()
176
- # return reviews2, total_reviews2
177
 
178
  css="footer {visibility: hidden}"
179
  # Applying style to highlight the maximum value in each row
@@ -191,14 +174,61 @@ with gr.Blocks(css=css) as demo:
191
  #run_actr()
192
  submit = gr.Button(value=".")
193
  submit.click(ccogsphere, [name, rate, celsci], [data, count])
194
- demo.load(get_latest_reviews, celsci, [data, count])
195
- #@name.change(inputs=name, outputs=celsci,_js="window.location.reload()")
196
- #@rate.change(inputs=rate, outputs=name,_js="window.location.reload()")
197
- #@celsci.change(inputs=celsci, outputs=rate,_js="window.location.reload()")
198
 
199
- #def secwork(celsci):
200
  #if name=="abc":
201
  #run_code()
202
- #demo.load(get_latest_reviews, celsci, [data, count])
203
  #return "Hello " + name + "!"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
204
  demo.launch()
 
9
  import math
10
  import json
11
 
12
+ import sqlite3
13
+ import huggingface_hub
14
  #import pandas as pd
15
+ import shutil
16
  import os
17
  import datetime
18
  from apscheduler.schedulers.background import BackgroundScheduler
 
21
  import time
22
  #import requests
23
 
24
+ from huggingface_hub import hf_hub_download
 
25
 
 
 
26
  #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./reviews.csv")
27
 
28
+ from huggingface_hub import login
29
  from datasets import load_dataset
30
 
31
  #dataset = load_dataset("csv", data_files="./data.csv")
32
 
33
 
34
+ DB_FILE = "./reviewsitr.db"
35
 
36
+ TOKEN = os.environ.get('HF_KEY')
37
 
38
+ repo = huggingface_hub.Repository(
39
+ local_dir="data",
40
+ repo_type="dataset",
41
+ clone_from="CognitiveScience/csdhdata",
42
+ use_auth_token=TOKEN
43
+ )
44
+ repo.git_pull()
45
 
46
  #TOKEN2 = HF_TOKEN
47
 
 
53
 
54
  # Create table if it doesn't already exist
55
 
56
+ db = sqlite3.connect(DB_FILE)
57
+ try:
58
+ db.execute("SELECT * FROM reviews").fetchall()
59
+ #db.execute("SELECT * FROM reviews2").fetchall()
60
+
61
+ db.close()
62
+ except sqlite3.OperationalError:
63
+ db.execute(
64
+ '''
65
+ CREATE TABLE reviews (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
66
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
67
+ name TEXT, view TEXT, duration TEXT)
68
+ ''')
69
+ db.commit()
70
+ db.close()
71
+
72
+ db = sqlite3.connect(DB_FILE)
73
+
74
+ def get_latest_reviews(db: sqlite3.Connection):
75
+ reviews = db.execute("SELECT * FROM reviews ORDER BY id DESC limit 100").fetchall()
76
+ total_reviews = db.execute("Select COUNT(id) from reviews").fetchone()[0]
77
+ reviews = pd.DataFrame(reviews, columns=["id", "date_created", "name", "view", "duration"])
 
 
 
 
 
 
 
 
78
  return reviews, total_reviews
79
 
80
+ def get_latest_reviews2(db: sqlite3.Connection):
81
+ reviews2 = db.execute("SELECT * FROM reviews2 ORDER BY id DESC limit 100").fetchall()
82
+ total_reviews2 = db.execute("Select COUNT(id) from reviews2").fetchone()[0]
83
+ reviews2 = pd.DataFrame(reviews2, columns=["id","title", "link","channel", "description", "views", "uploaded", "duration", "durationString"])
84
+ return reviews2, total_reviews2
85
 
86
  def ccogsphere(name: str, rate: int, celsci: str):
87
+ db = sqlite3.connect(DB_FILE)
88
+ cursor = db.cursor()
89
 
90
  #try:
91
  celsci2=celsci.split()
 
99
  duration = str(row["duration"])
100
  print(view, duration)
101
  #celsci=celsci+celsci2
102
+ cursor.execute("INSERT INTO reviews(name, view, duration) VALUES(?,?,?)", [celsci+str(index+1), view, duration])
103
+ db.commit()
104
+
105
+ reviews, total_reviews = get_latest_reviews(db)
106
+ db.close()
 
 
107
  r = requests.post(url='https://ccml-persistent-data2.hf.space/api/predict/', json={"data": [celsci + " ", celsci2]})
108
 
109
  return reviews, total_reviews
 
130
  try:
131
  result=ecf(inp)
132
  df=pd.DataFrame.from_dict(result["videos"])
133
+ except sqlite3.OperationalError:
134
+ print ("db error")
 
135
 
136
+ df=df.drop(df.columns[4], axis=1)
137
 
138
+ db = sqlite3.connect(DB_FILE)
139
  #cursor = db.cursor()
140
  #cursor.execute("INSERT INTO reviews2(title, link, thumbnail,channel, description, views, uploaded, duration, durationString) VALUES(?,?,?,?,?,?,?,?,?)", [title, link, thumbnail,channel, description, views, uploaded, duration, durationString])
141
+ df.to_sql('reviews2', db, if_exists='replace', index=False)
142
 
143
  #db.commit()
144
+ reviews2, total_reviews2 = get_latest_reviews(db)
145
+ db.close()
146
  #print ("print000", total_reviews2,reviews2)
 
 
 
147
  return reviews2, total_reviews2
148
 
149
 
150
+ def load_data():
151
+ db = sqlite3.connect(DB_FILE)
152
+ reviews, total_reviews = get_latest_reviews(db)
153
+ db.close()
154
+ return reviews, total_reviews
155
+ def load_data2():
156
+ db = sqlite3.connect(DB_FILE)
157
+ reviews2, total_reviews2 = get_latest_reviews2(db)
158
+ db.close()
159
+ return reviews2, total_reviews2
160
 
161
  css="footer {visibility: hidden}"
162
  # Applying style to highlight the maximum value in each row
 
174
  #run_actr()
175
  submit = gr.Button(value=".")
176
  submit.click(ccogsphere, [name, rate, celsci], [data, count])
177
+ demo.load(load_data, None, [data, count])
178
+ @name.change(inputs=name, outputs=celsci,_js="window.location.reload()")
179
+ @rate.change(inputs=rate, outputs=name,_js="window.location.reload()")
180
+ @celsci.change(inputs=celsci, outputs=rate,_js="window.location.reload()")
181
 
182
+ def secwork(name):
183
  #if name=="abc":
184
  #run_code()
185
+ load_data()
186
  #return "Hello " + name + "!"
187
+ def backup_db():
188
+ shutil.copyfile(DB_FILE, "./reviews.db")
189
+ db = sqlite3.connect(DB_FILE)
190
+ reviews = db.execute("SELECT * FROM reviews").fetchall()
191
+ pd.DataFrame(reviews).to_csv("./reviews.csv", index=False)
192
+ print("updating db")
193
+ repo.push_to_hub(blocking=False, commit_message=f"Updating data at {datetime.datetime.now()}")
194
+
195
+ def backup_db_csv():
196
+ shutil.copyfile(DB_FILE, "./reviews2.db")
197
+ db = sqlite3.connect(DB_FILE)
198
+ reviews = db.execute("SELECT * FROM reviews").fetchall()
199
+ pd.DataFrame(reviews).to_csv("./reviews2.csv", index=False)
200
+ print("updating db csv")
201
+ dataset = load_dataset("csv", data_files="./reviews2.csv")
202
+ repo.push_to_hub("CognitiveScience/csdhdata", blocking=False) #, commit_message=f"Updating data-csv at {datetime.datetime.now()}")
203
+ #path1=hf_hub_url()
204
+ #print (path1)
205
+ #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./*.csv")
206
+ #hf_hub_download(repo_id="CognitiveScience/csdhdata", filename="./*.db")
207
+ #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./*.md")
208
+ #hf_hub_download(repo_id="CognitiveScience/csdhdata", filename="./*.md")
209
+
210
+
211
+ #def load_data2():
212
+ # db = sqlite3.connect(DB_FILE)
213
+ # reviews, total_reviews = get_latest_reviews(db)
214
+ # #db.close()
215
+ # demo.load(load_data,None, [reviews, total_reviews])
216
+ # #return reviews, total_reviews
217
+
218
+ #scheduler1 = BackgroundScheduler()
219
+ #scheduler1.add_job(func=run_actr, trigger="interval", seconds=10)
220
+ #scheduler1.start()
221
+
222
+ scheduler1 = BackgroundScheduler()
223
+ scheduler1.add_job(func=load_data, trigger="interval", seconds=5)
224
+ scheduler1.start()
225
+
226
+ scheduler2 = BackgroundScheduler()
227
+ scheduler2.add_job(func=backup_db, trigger="interval", seconds=3633)
228
+ scheduler2.start()
229
+
230
+ scheduler3 = BackgroundScheduler()
231
+ scheduler3.add_job(func=backup_db_csv, trigger="interval", seconds=3666)
232
+ scheduler3.start()
233
+
234
  demo.launch()