File size: 721 Bytes
928f123
7e4123a
 
928f123
 
 
046ea77
 
 
 
 
 
 
 
 
928f123
7e4123a
928f123
 
 
 
7e4123a
928f123
 
 
 
 
 
 
7e4123a
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import datasets
import pandas as pd

from huggingface_hub import create_repo
from huggingface_hub.utils import HfHubHTTPError

def create_hf_hub(
    repo_id, hf_token
):
    try:
        create_repo(repo_id, repo_type="dataset", token=hf_token)
    except HfHubHTTPError as e:
        return False
    return True

def push_to_hf_hub(
	ds, repo_id, hf_token, append=True
):
    exist = False

    try:
        create_repo(repo_id, repo_type="dataset", token=hf_token)
    except HfHubHTTPError as e:
        exist = True
    
    if exist and append:
        existing_ds = datasets.load_dataset(repo_id)
        ds = datasets.concatenate_datasets([existing_ds['train'], ds])

    ds.push_to_hub(repo_id, token=hf_token)