Update main.py

main.py

@@ -1,14 +1,19 @@
 from fastapi import FastAPI
 import uvicorn
 
+
+import streamlit as st
 import pandas as pd
 import numpy as np
-import pickle
-import rasterio
-import h5py
-from skimage.morphology import disk
-from geopy.extra.rate_limiter import RateLimiter
-from geopy.geocoders import Nominatim
+import requests
+from urllib.parse import urlparse, quote
+import re
+from bs4 import BeautifulSoup
+import time
+from joblib import Parallel, delayed
+from nltk import ngrams
+from googlesearch import search
+
 
 app = FastAPI()
 
@@ -17,118 +22,92 @@ app = FastAPI()
 #Root endpoints
 @app.get("/")
 def root():
-    return {"API": "Hail Docker Data"}
-    
-def geocode_address(address):
+    return {"API": "AdressScrap"}
 
+def jaccard_similarity(string1, string2,n = 2, normalize=True):
     try:
-        address2 = address.replace(' ', '+').replace(',', '%2C')
-        df = pd.read_json(
-            f'https://geocoding.geo.census.gov/geocoder/locations/onelineaddress?address={address2}&benchmark=2020&format=json')
-        results = df.iloc[:1, 0][0][0]['coordinates']
-        lat, lon = results['y'], results['x']
+        if normalize:
+           string1,string2= normalize_string(string1),normalize_string(string2)
+           
+        grams1 = set(ngrams(string1, n))
+        grams2 = set(ngrams(string2, n))
+        similarity = len(grams1.intersection(grams2)) / len(grams1.union(grams2))
     except:
-        geolocator = Nominatim(user_agent='GTA Lookup')
-        geocode = RateLimiter(geolocator.geocode, min_delay_seconds=2)
-        location = geolocator.geocode(address)
-        lat, lon = location.latitude, location.longitude
+        similarity=0
         
-    return lat, lon
-
-def get_hail_data(address, start_date, end_date, radius_miles, get_max):
-
-    resolution=1 # mrms 1 and hrrr is 3
-    radius = int(np.ceil(radius_miles*1.6/resolution))
-
-    
-    start_date = pd.Timestamp(str(start_date)).strftime('%Y%m%d')
-    end_date = pd.Timestamp(str(end_date)).strftime('%Y%m%d')
-    date_years = pd.date_range(start=start_date, end=end_date, freq='M')
-    date_range_days = pd.date_range(start_date, end_date)
-    years = list(set([d.year for d in date_years]))
-
-    if len(years) == 0:
-        years = [pd.Timestamp(start_date).year]
+    if string2=='did not extract address':
+        similarity=0
         
-    # Geocode Address
-    lat, lon= geocode_address(address)
+    return similarity
 
-    # Convert Lat Lon to row & col on Array
-  
-    transform = pickle.load(open('Data/transform_mrms.pkl', 'rb'))
+def jaccard_sim_split_word_number(string1,string2):
+    numbers1 = ' '.join(re.findall(r'\d+', string1))
+    words1 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string1))
     
-    row, col = rasterio.transform.rowcol(transform, lon, lat)
-
-   
-
-    files = [
-        'Data/2023_hail.h5',
-        'Data/2022_hail.h5',
-        'Data/2021_hail.h5',
-        'Data/2020_hail.h5'
-    ]
-
-    files_choosen = [i for i in files if any(i for j in years if str(j) in i)]
-
-    # Query and Collect H5 Data
-    all_data = []
-    all_dates = []
-    for file in files_choosen:
-        with h5py.File(file, 'r') as f:
-            # Get Dates from H5
-            dates = f['dates'][:]
-            date_idx = np.where((dates >= int(start_date))
-                                & (dates <= int(end_date)))[0]
-
-            # Select Data by Date and Radius
-            dates = dates[date_idx]
-            data = f['hail'][date_idx, row-radius_miles:row +
-                             radius_miles+1, col-radius_miles:col+radius_miles+1]
-
-            all_data.append(data)
-            all_dates.append(dates)
-
-    data_all = np.vstack(all_data)
-    dates_all = np.concatenate(all_dates)
-
-    # Convert to Inches
-    data_mat = np.where(data_all < 0, 0, data_all)*0.0393701
+    numbers2 = ' '.join(re.findall(r'\d+', string2))
+    words2 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string2))   
+    
+    number_similarity=jaccard_similarity(numbers1,numbers2)
+    words_similarity=jaccard_similarity(words1,words2)
+    return (number_similarity+words_similarity)/2
 
-    # Get Radius of Data
-    disk_mask = np.where(disk(radius_miles) == 1, True, False)
-    data_mat = np.where(disk_mask, data_mat, -1).round(3)
+def extract_website_domain(url):
+    parsed_url = urlparse(url)
+    return parsed_url.netloc
 
-    # Process to DataFrame
-    # Find Max of Data
-    if get_max == True:
-        data_max = np.max(data_mat, axis=(1, 2))
-        df_data = pd.DataFrame({'Date': dates_all,
-                               'Hail_max': data_max})
-    # Get all Data
-    else:
-        data_all = list(data_mat)
-        df_data = pd.DataFrame({'Date': dates_all,
-                               'Hail_all': data_all})
 
-    df_data['Date'] = pd.to_datetime(df_data['Date'], format='%Y%m%d')
-    df_data = df_data.set_index('Date')
+def google_address(address):     
+    all_data=[i for i in search(address, ssl_verify=False, advanced=True,
+                                num_results=11)]
+    
+    
+    df=pd.DataFrame({'Title':[i.title for i in all_data],
+                     'Link':[i.url for i in all_data],
+                     'Description':[i.description for i in all_data],})
+    
+    df=df.query("Title==Title")
+    df['Link']=df['Link'].str.replace('/www.','https://www.')
+    
+    # df['Description']=df['Description'].bfill()
+    df['Address Output']=df['Title'].str.extract(r'(.+? \d{5})').fillna("**DID NOT EXTRACT ADDRESS**")
+    
+    df['Link']=[i[7:i.find('&sa=')] for i in df['Link']]
+    df['Website'] = df['Link'].apply(extract_website_domain)
+    
+    df['Square Footage']=df['Description'].str.extract(r"((\d+) Square Feet|(\d+) sq. ft.|(\d+) sqft|(\d+) Sq. Ft.|(\d+) sq|(\d+(?:,\d+)?) Sq\. Ft\.|(\d+(?:,\d+)?) sq)")[0]
+    try:
+        df['Square Footage']=df['Square Footage'].replace({',':''},regex=True).str.replace(r'\D', '')
+    except:
+        pass
+    df['Beds']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"(\d+) bed")
+    
+    
+    df['Baths']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"((\d+) bath|(\d+(?:\.\d+)?) bath)")[0]
+    df['Baths']=df['Baths'].str.extract(r'([\d.]+)').astype(float)
+    
+    df['Year Built']=df['Description'].str.extract(r"built in (\d{4})")
+    
+    df['Match Percent']=[jaccard_sim_split_word_number(address,i)*100 for i in df['Address Output']]
+    df['Google Search Result']=[*range(1,df.shape[0]+1)]
+   
+    # df_final=df[df['Address Output'].notnull()]
+    # df_final=df_final[(df_final['Address Output'].str.contains(str(address_number))) & (df_final['Address Output'].str.contains(str(address_zip)))]
 
-    df_data = df_data.reindex(date_range_days, fill_value=0).reset_index().rename(
-        columns={'index': 'Date'})
-    df_data['Date'] = df_data['Date'].dt.strftime('%Y-%m-%d')
+    df.insert(0,'Address Input',address)
+    
+    return df
 
-    return df_data
    
 
 @app.get('/Hail_Docker_Data')
-async def predict(address: str, start_date: str, end_date: str, radius_miles: int, get_max: bool):
-
-    try:
-        results = get_hail_data(address, start_date,
-                                end_date, radius_miles, get_max)
+async def predict(address: str):
+    try: 
+        results= google_address(addresses)
+        results=results[['Address Input', 'Address Output','Match Percent','Website','Square Footage', 'Beds', 'Baths', 'Year Built',
+                     'Link','Google Search Result', 'Description' ]]
     except:
-        results = pd.DataFrame({'Date': ['error'], 'Hail_max': ['error']})
-
-    return results.to_json()
+        results= pd.DataFrame({'Address Input':[addresses]})
+        
+    return results