Spaces:
Running
Running
File size: 2,303 Bytes
11550ac 55d18b1 11550ac 6f483b1 11550ac 55d18b1 8c4b1f7 11550ac fd0073b 11550ac 8c4b1f7 11550ac 8c4b1f7 6f483b1 8c4b1f7 11550ac 8c4b1f7 11550ac 6f483b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
import os
import json
import tempfile
import logging
from streamlit.delta_generator import DeltaGenerator
import streamlit as st
from huggingface_hub import HfApi, CommitInfo
from dataset.download import dataset_id
# get a global var for logger accessor in this module
LOG_LEVEL = logging.DEBUG
g_logger = logging.getLogger(__name__)
g_logger.setLevel(LOG_LEVEL)
def push_observation(image_hash:str, api:HfApi, enable_push:False) -> CommitInfo:
'''
push one observation to the Hugging Face dataset
'''
# get the observation
observation = st.session_state.public_observations.get(image_hash)
if observation is None:
msg = f"Could not find observation with hash {image_hash}"
g_logger.error(msg)
st.error(msg)
return None
# convert to json
metadata_str = json.dumps(observation) # doesn't work yet, TODO
st.toast(f"Uploading observation: {metadata_str}", icon="🦭")
g_logger.info(f"Uploading observation: {metadata_str}")
# write to temp file so we can send it (why is this not using context mgr?)
f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
f.write(metadata_str)
f.close()
#st.info(f"temp file: {f.name} with metadata written...")
path_in_repo = f"metadata/{observation['author_email']}/{observation['image_md5']}.json"
msg = f"fname: {f.name} | path: {path_in_repo}"
print(msg)
st.warning(msg)
if enable_push:
rv = api.upload_file(
path_or_fileobj=f.name,
path_in_repo=path_in_repo,
repo_id=dataset_id,
repo_type="dataset",
)
print(rv)
msg = f"observation attempted tx to repo happy walrus: {rv}"
g_logger.info(msg)
st.info(msg)
else:
rv = None # temp don't send anything
return rv
def push_all_observations(enable_push:bool=False):
'''
open an API connection to Hugging Face, and push all observation one by one
'''
# get huggingface api
token = os.environ.get("HF_TOKEN", None)
api = HfApi(token=token)
# iterate over the list of observations
for hash in st.session_state.public_observations.keys():
rv = push_observation(hash, api, enable_push=enable_push) |