PR conflict resolution plus making HF upload more generic

.github/workflows/sync_to_hub.yml

@@ -0,0 +1,20 @@
+name: Sync to Hugging Face hub
+
+on:
+  push:
+    branches: [master]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push --force https://gagan3012:$HF_TOKEN@huggingface.co/spaces/gagan3012/t5-summarisation

.gitignore

@@ -97,4 +97,5 @@ summarization-dagshub/
 /models
 default/
 artifacts/
-mlruns/
+mlruns/
+hf_model/

README.md

@@ -1,3 +1,13 @@
+---
+title: T5-Summarisation
+emoji: ✌
+colorFrom: yellow
+colorTo: red
+sdk: streamlit
+app_file: app.py
+pinned: false
+---
+
 summarization
 ==============================
 

dvc.lock

@@ -10,8 +10,8 @@ stages:
       md5: 6069153a075b00dfb6d9e0843dd2da89
       size: 52739
     - path: model_params.yml
-      md5: 9fcf006ee30f2b751078598a3fba9bb5
-      size: 235
+      md5: 1bf2edf25e851cc9cd3be75fbd9905a3
+      size: 177
     - path: src/models/train_model.py
       md5: f7d1121426c3d5530c2b9697cb7ac74a
       size: 951
@@ -21,7 +21,7 @@ stages:
       size: 243476333
       nfiles: 5
     - path: reports/training_metrics.csv
-      md5: 0b6c1518aed802bea976e883caac2a90
+      md5: 3b309def91a32e521acd23b163742522
       size: 320
   eval:
     cmd: python src/models/evaluate_model.py
@@ -30,8 +30,8 @@ stages:
       md5: 3eec94ac211c76363a3d968663b82d02
       size: 39574
     - path: model_params.yml
-      md5: 9fcf006ee30f2b751078598a3fba9bb5
-      size: 235
+      md5: 1bf2edf25e851cc9cd3be75fbd9905a3
+      size: 177
     - path: models
       md5: fc37870a93db61b94af9f0847577f09b.dir
       size: 243476333
@@ -41,7 +41,7 @@ stages:
       size: 705
     outs:
     - path: reports/evaluation_metrics.csv
-      md5: a5fa12e6df10884217614c007d146a26
+      md5: eaa3bf017026aa1be31560f308fff78e
       size: 2122
   process_data:
     cmd: python src/data/process_data.py
@@ -88,9 +88,18 @@ stages:
       size: 243476333
       nfiles: 5
     - path: src/visualization/visualize.py
-      md5: a71303fef593a9fd275fc4964623baf8
-      size: 814
-    outs:
-    - path: reports/visualization_metrics.txt
-      md5: fd7b6bb170dbaa9ef1076bc8be7e7593
-      size: 2144
+      md5: 4226e4148abb5ac186c0ab8c1d87b228
+      size: 671
+  push_to_hf_hub:
+    cmd: python src/models/hf_upload.py
+    deps:
+    - path: model_params.yml
+      md5: 1bf2edf25e851cc9cd3be75fbd9905a3
+      size: 177
+    - path: models
+      md5: fc37870a93db61b94af9f0847577f09b.dir
+      size: 243476333
+      nfiles: 5
+    - path: src/models/hf_upload.py
+      md5: a953816a3eb7bef702313544103a1c11
+      size: 1290

model_params.yml

@@ -9,6 +9,3 @@ num_workers: 2
 model_dir: models
 metric: rouge
 source_dir: src
-visualise: True
-hf_username: gagan3012
-upload_to_hf: False

reports/evaluation_metrics.csv

@@ -1,37 +1,37 @@
 Name,Value,Timestamp,Step
-"Rouge_1 Low Precision",0.23786550570641482,1628194352980,1
-"Rouge_1 Low recall",0.23355396379384713,1628194352980,1
-"Rouge_1 Low F1",0.23602599457077003,1628194352980,1
-"Rouge_1 Mid Precision",0.3569471852499436,1628194352980,1
-"Rouge_1 Mid recall",0.31915939075819916,1628194352980,1
-"Rouge_1 Mid F1",0.3317618573023773,1628194352980,1
-"Rouge_1 High Precision",0.4726861301480842,1628194352980,1
-"Rouge_1 High recall",0.4019654200001146,1628194352980,1
-"Rouge_1 High F1",0.4298956952594035,1628194352980,1
-"Rouge_2 Low Precision",0.06184772400193972,1628194352980,1
-"Rouge_2 Low recall",0.05626972412346313,1628194352980,1
-"Rouge_2 Low F1",0.058680298802341754,1628194352980,1
-"Rouge_2 Mid Precision",0.1367034298993256,1628194352980,1
-"Rouge_2 Mid recall",0.11953160646342464,1628194352980,1
-"Rouge_2 Mid F1",0.12485064123505887,1628194352980,1
-"Rouge_2 High Precision",0.22739029631016827,1628194352980,1
-"Rouge_2 High recall",0.18851628169809986,1628194352980,1
-"Rouge_2 High F1",0.20306657551189072,1628194352980,1
-"Rouge_L Low Precision",0.18248956154159507,1628194352980,1
-"Rouge_L Low recall",0.18048774357814204,1628194352980,1
-"Rouge_L Low F1",0.18151380309623336,1628194352980,1
-"Rouge_L Mid Precision",0.2614974838710314,1628194352980,1
-"Rouge_L Mid recall",0.24286688705755238,1628194352980,1
-"Rouge_L Mid F1",0.24674586991996245,1628194352980,1
-"Rouge_L High Precision",0.3574471638807763,1628194352980,1
-"Rouge_L High recall",0.30836083808542225,1628194352980,1
-"Rouge_L High F1",0.32385446385474176,1628194352980,1
-"rougeLsum Low Precision",0.21468633089019287,1628194352980,1
-"rougeLsum Low recall",0.2057771050364415,1628194352980,1
-"rougeLsum Low F1",0.21170611912426093,1628194352980,1
-"rougeLsum Mid Precision",0.3060593850789648,1628194352980,1
-"rougeLsum Mid recall",0.27733553744690076,1628194352980,1
-"rougeLsum Mid F1",0.28530501988436374,1628194352980,1
-"rougeLsum High Precision",0.4094614601758424,1628194352980,1
-"rougeLsum High recall",0.34640369291505535,1628194352980,1
-"rougeLsum High F1",0.36454440079714096,1628194352980,1
+"Rouge_1 Low Precision",0.23786550570641482,1628587253223,1
+"Rouge_1 Low recall",0.23355396379384713,1628587253223,1
+"Rouge_1 Low F1",0.23602599457077003,1628587253223,1
+"Rouge_1 Mid Precision",0.3569471852499436,1628587253223,1
+"Rouge_1 Mid recall",0.31915939075819916,1628587253223,1
+"Rouge_1 Mid F1",0.3317618573023773,1628587253223,1
+"Rouge_1 High Precision",0.4726861301480842,1628587253223,1
+"Rouge_1 High recall",0.4019654200001146,1628587253223,1
+"Rouge_1 High F1",0.4298956952594035,1628587253223,1
+"Rouge_2 Low Precision",0.06184772400193972,1628587253223,1
+"Rouge_2 Low recall",0.05626972412346313,1628587253223,1
+"Rouge_2 Low F1",0.058680298802341754,1628587253223,1
+"Rouge_2 Mid Precision",0.1367034298993256,1628587253223,1
+"Rouge_2 Mid recall",0.11953160646342464,1628587253223,1
+"Rouge_2 Mid F1",0.12485064123505887,1628587253223,1
+"Rouge_2 High Precision",0.22739029631016827,1628587253223,1
+"Rouge_2 High recall",0.18851628169809986,1628587253223,1
+"Rouge_2 High F1",0.20306657551189072,1628587253223,1
+"Rouge_L Low Precision",0.18248956154159507,1628587253223,1
+"Rouge_L Low recall",0.18048774357814204,1628587253223,1
+"Rouge_L Low F1",0.18151380309623336,1628587253223,1
+"Rouge_L Mid Precision",0.2614974838710314,1628587253223,1
+"Rouge_L Mid recall",0.24286688705755238,1628587253223,1
+"Rouge_L Mid F1",0.24674586991996245,1628587253223,1
+"Rouge_L High Precision",0.3574471638807763,1628587253223,1
+"Rouge_L High recall",0.30836083808542225,1628587253223,1
+"Rouge_L High F1",0.32385446385474176,1628587253223,1
+"rougeLsum Low Precision",0.21468633089019287,1628587253223,1
+"rougeLsum Low recall",0.2057771050364415,1628587253223,1
+"rougeLsum Low F1",0.21170611912426093,1628587253223,1
+"rougeLsum Mid Precision",0.3060593850789648,1628587253223,1
+"rougeLsum Mid recall",0.27733553744690076,1628587253223,1
+"rougeLsum Mid F1",0.28530501988436374,1628587253223,1
+"rougeLsum High Precision",0.4094614601758424,1628587253223,1
+"rougeLsum High recall",0.34640369291505535,1628587253223,1
+"rougeLsum High F1",0.36454440079714096,1628587253223,1

reports/training_metrics.csv

@@ -1,9 +1,9 @@
 Name,Value,Timestamp,Step
-"val_loss",2.615034580230713,1628194199660,0
-"epoch",0,1628194199660,0
-"val_loss",2.6141018867492676,1628194229556,1
-"epoch",1,1628194229556,1
-"val_loss",2.6132164001464844,1628194259447,2
-"epoch",2,1628194259447,2
-"val_loss",2.612450361251831,1628194289914,3
-"epoch",3,1628194289914,3
+"val_loss",2.615034580230713,1628591864766,0
+"epoch",0,1628591864766,0
+"val_loss",2.6141018867492676,1628591893945,1
+"epoch",1,1628591893945,1
+"val_loss",2.6132164001464844,1628591923101,2
+"epoch",2,1628591923101,2
+"val_loss",2.612450361251831,1628591951319,3
+"epoch",3,1628591951319,3

src/models/hf_upload.py

@@ -7,35 +7,34 @@ from model import Summarization
 from huggingface_hub import HfApi, Repository
 
 
-def upload(upload_model, model_name):
+def upload(model_to_upload, model_name):
     hf_username = input("Enter your HuggingFace username:")
-    hf_password = getpass("Enter your HuggingFace password:")
-    if Path("./models").exists():
-        shutil.rmtree("./models")
-    token = HfApi().login(username=hf_username, password=hf_password)
-    del hf_password
-    model_url = HfApi().create_repo(token=token, name=model_name, exist_ok=True)
+    hf_token = getpass("Enter your HuggingFace token:")
+    model_url = HfApi().create_repo(token=hf_token, name=model_name, exist_ok=True)
     model_repo = Repository(
-        "./model",
+        "./hf_model",
         clone_from=model_url,
-        use_auth_token=token,
+        use_auth_token=hf_token,
         git_email=f"{hf_username}@users.noreply.huggingface.co",
         git_user=hf_username,
     )
 
+    del hf_token
     readme_txt = f"""
             ---
             Summarisation model {model_name}
             """.strip()
 
     (Path(model_repo.local_dir) / "README.md").write_text(readme_txt)
-    upload_model.save_model()
     commit_url = model_repo.push_to_hub()
 
     print("Check out your model at:")
     print(commit_url)
     print(f"https://huggingface.co/{hf_username}/{model_name}")
 
+    if Path("./hf_model").exists():
+        shutil.rmtree("./hf_model")
+
 
 if __name__ == "__main__":
     with open("model_params.yml") as f:
@@ -44,4 +43,4 @@ if __name__ == "__main__":
     model = Summarization()
     model.load_model(model_dir="./models")
 
-    upload(upload_model=model, model_name=params["name"])
+    upload(model_to_upload=model, model_name=params["name"])