Create FineTuning_Lang_models.py

FineTuning_Lang_models.py

@@ -0,0 +1,40 @@
+import os
+import pandas as pd
+import streamlit as st
+HG_DIR = '/nlp/scr/msuzgun/cache_extra/huggingface'
+# Specify HG cache dirs -- currently use only for 2.7b model
+os.environ['TRANSFORMERS_CACHE'] = f'{HG_DIR}/transformers'
+os.environ['HF_HOME'] = HG_DIR
+
+## Import relevant libraries and dependencies
+!pip install datasets
+!pip install Transformers
+!pip install streamlit
+# Pretty print
+from pprint import pprint
+# Datasets load_dataset function
+from datasets import load_dataset
+# Transformers Autokenizer
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
+# Standard PyTorch DataLoader
+from torch.utils.data import DataLoader
+
+
+
+dataset_dict = load_dataset('HUPD/hupd',
+    name='sample',
+    data_files="https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather", 
+    cache_dir ='/u/scr/nlp/data/HUPD',
+    icpr_label=None,
+    train_filing_start_date='2016-01-01',
+    train_filing_end_date='2016-01-31',
+    val_filing_start_date='2017-01-01',
+    val_filing_end_date='2017-01-31',
+)
+
+df = pd.DataFrame.from_dict(dataset_dict["train"])
+
+# Create a DataFrame object from list
+df = pd.DataFrame(df,columns =['patent_number','decision', 'abstract', 'claims','filing_date'])
+st.dataframe(df)