Spaces:
Sleeping
Sleeping
File size: 2,873 Bytes
e21a7d7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import os
import pandas as pd
import streamlit as st
from google.cloud import bigquery
from utils import load_gcp_credentials, upload_to_bigquery, authenticate_bigquery
from html_templates import logo, BigQuery_upload_title, table_id_placeholder, uploader, button_styles, tooltip_message_bq, recon_title
import json
st.markdown(logo, unsafe_allow_html=True)
st.logo("alerter_4.jpeg")
st.markdown(BigQuery_upload_title, unsafe_allow_html=True)
st.write("")
st.write("")
#Tooltip
st.markdown(tooltip_message_bq, unsafe_allow_html=True)
# Display input for BigQuery table ID
st.markdown(table_id_placeholder, unsafe_allow_html=True)
table_id = st.text_input("", key="table_id_name", help="Enter the BigQuery table ID where you want to upload the data. Example: project_id.dataset_id.table_id")
# Retrieve the service account credentials
gcp_credentials = load_gcp_credentials()
if gcp_credentials:
bigquery_creds = authenticate_bigquery()
client = bigquery.Client(credentials=bigquery_creds)
st.markdown(uploader, unsafe_allow_html=True)
uploaded_file = st.file_uploader("", type=["csv", "xlsx"], help = "Upload the CSV/Excel file you need to upload in the BQ table")
if uploaded_file is not None:
if uploaded_file.type in ['application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']:
df = pd.read_excel(uploaded_file)
else:
df = pd.read_csv(uploaded_file)
# Preview the DataFrame
st.write(df)
# Convert DataFrame to JSON
try:
json_data = df.to_json(orient="records", lines=True)
st.write("")
except Exception as e:
st.error(f"Error converting DataFrame to JSON: {e}")
st.stop()
# Submit to BigQuery
st.markdown(button_styles, unsafe_allow_html = True)
if st.button("Submit"):
try:
# Configure BigQuery job
job_config = bigquery.LoadJobConfig(
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
)
# Parse JSON data into lines
json_lines = [json.loads(line) for line in json_data.splitlines()]
# Load JSON data into BigQuery
job = client.load_table_from_json(
json_lines, table_id, job_config=job_config
)
job.result() # Wait for the job to complete
records_inserted = job.output_rows
record_word = "record" if records_inserted == 1 else "records"
st.success(f"{records_inserted} {record_word} inserted into BigQuery")
except Exception as e:
st.error(f"Error uploading data to BigQuery: {e}") |