app.py

import streamlit as st
import numpy as np
import pandas as pd
import geopandas as gpd
import gspread
import folium
import requests as r
from datetime import datetime
from datetime import date
from bs4 import BeautifulSoup as bs
from shapely.geometry import Point
from google.oauth2.service_account import Credentials
from streamlit_folium import folium_static
import pickle
import os
import re
#API='80ccf867-5353-4474-a01c-d1299496eca7'
API='deb65d93-a965-4d1f-a09e-5a047ea959d7'



@st.cache
def make_date(a):
    try:
        return datetime.strptime(a, "%d.%m.%Y").date()
    except:
        return date(2000, 3, 14)


@st.cache
def make_date_new(a):
    if type(a)==date:
        return a
    try:
        return datetime.strptime(a, "%Y-%m-%d").date()
    except:
        return date(2000, 3, 14)


def make_area(a):
    try:
        return float(a)
    except:
        return 0.0

def make_point(a):
    try:
        return Point(a)
    except:
        try:
            coordss=re.findall("\d+\.\d+", a)
            to_point=[]
            for i in coordss:
                to_point.append(float(i))
            return Point(to_point)
        except:
            return a

@st.cache
def GET_COORDS_YANDEX(adress):
    ADRESS=adress.replace(' ', '+')
    result=r.get(f'https://geocode-maps.yandex.ru/1.x/?apikey={API}&geocode={ADRESS}&results=1')
    res_b_xml=bs(result.content, "lxml")
    try:
        coords=res_b_xml.find('pos').text.split()
        res=[float(a) for a in coords]
        return Point(res)
    except:
        return None



def first_try(data):
    old_data=[data[0]]
    len_old_data=1
    return old_data, len_old_data



def import_data():
    scopes = [
        'https://www.googleapis.com/auth/spreadsheets',
        'https://www.googleapis.com/auth/drive'
    ]
    
    credentials = Credentials.from_service_account_file(
        'test-project-for-telegram-bot-bcb5e887f016.json',
        scopes=scopes
    )
    
    gc = gspread.authorize(credentials)
    spr = gc.open_by_url("https://docs.google.com/spreadsheets/d/1Ol9039HJ99Q8xVVp1Wwz5iBdURxEKHLIcEvyqUc2pZM/edit#gid=642560994")
    wks=spr.worksheet('Консолидированный список')
    new_data=wks.get_all_values()
    new_data[0].pop(-1)
    new_data[0].pop(-1)
    #with open('old_data.pickle', 'rb') as f:
    #    if os.stat('old_data.pickle').st_size == 0:
    #        old_data, len_old_data=first_try(new_data)
    #    else:
    #        old_data=pickle.load(f)
    #        len_old_data=len(old_data)
    if os.stat('old_data.csv').st_size == 0:
        old_data, len_old_data=first_try(new_data)
    else:
        DD=pd.read_csv('old_data.csv')
        old_data=[DD.columns]
        for a in DD.values:
            old_data.append(a)
        len_old_data=len(old_data)
    list_to_geocode=new_data[len_old_data::]
    i=0
    for lead in list_to_geocode:
        lead.pop(-1)
        lead.pop(-1)
        lead.append(GET_COORDS_YANDEX(lead[8]))
        #lead.append([55.87890+i, 37.71943-i])
        lead.append(make_date(lead[3]))
        #i+=.00001
    for lead in list_to_geocode:
        old_data.append(lead)
    if len(old_data[0])!=len(old_data[1]):
        old_data[0].append('coords1')
        old_data[0].append('date1')
    DD=pd.DataFrame(old_data[1::], columns=old_data[0])
    DD.to_csv('old_data.csv')
    return DD
    
        
OBJECTS=import_data()
OBJECTS=pd.DataFrame(
    {
        'комнатность':OBJECTS['Кол-во комнат'].values,
        'этаж':OBJECTS['Этаж'].values,
        'площадь':OBJECTS['Площадь '].values,
        'ремонт':OBJECTS['Ремонт'].values,
       'Диапазон предварительной оценки':OBJECTS['Предварительная оценка (рынок / выкуп) '].values,
       'Дата оценки':OBJECTS['date1'].values,
        'geometry':OBJECTS['coords1'].values
    }
)
OBJECTS['Дата оценки']=OBJECTS['Дата оценки'].apply(make_date_new)
OBJECTS['площадь']=OBJECTS['площадь'].apply(make_area)
OBJECTS['geometry']=OBJECTS['geometry'].apply(make_point)
#st.write(OBJECTS.head(3))
#GEO_OBJECTS=gpd.GeoDataFrame(OBJECTS)


with st.sidebar:
    st.header('Введите параметры оценок')
    min_date = st.date_input("введите минимальную дату оценки", date(2022, 10, 6))
    max_date = st.date_input("введите максимальную дату оценки", date(2023, 7, 6))
    rooms = st.multiselect('комантность', ['ст', '1', '2', '3', '4'], ['1', '2'])
    areamin, areamax = st.slider('Select a range of values', 0.0 , 200.0, (25.0, 75.0))
    start_adress = st.text_input("адресс текущей оценки")
    
data=[]
for A in OBJECTS.values:
    if (A[5]>min_date) and (A[5]<max_date) and (A[0] in rooms) and (A[2]>areamin) and (A[2]<areamax):
        data.append(A)

        
Data_to_plot=gpd.GeoDataFrame(data, columns=['комнатность',
                                             'этаж',
                                             'площадь',
                                             'ремонт',
                                             'Диапазон предварительной оценки',
                                             'Дата оценки',
                                             'geometry'])

st.write(Data_to_plot.head(3))

#Data_to_plot=GEO_OBJECTS.loc[(GEO_OBJECTS['Дата оценки']>min_date)&(GEO_OBJECTS['Дата оценки']<min_date)&(GEO_OBJECTS['комнатность'] in rooms)&(GEO_OBJECTS['площадь']>areamin)&(GEO_OBJECTS['площадь']<areamax)]


st.subheader('Карта оценок')
m = folium.Map(location=[55.87890, 37.71943], zoom_start=10, tiles='CartoDB positron')
fg4 = folium.map.FeatureGroup(name='pricing').add_to(m)

    
st.write(Data_to_plot.values[10][6])
if len(Data_to_plot)<500:
    for A in Data_to_plot.values:
        tag=''
        try:
            lat=A[6].y
            log=A[6].x
        except:
            continue
        for feature in A[:-2]:
            tag+=str(feature).replace('\"', '').replace("'","").replace("/"," ")+' '
        if len(tag)>40:
            tag=tag[5:]
    #    tag=f'хуувая квартира №{i}'
    #   tag+=str(A[-1])
        iframe = folium.IFrame(tag)
        popup = folium.Popup(iframe, min_width=100, max_width=100)
        Z=folium.Marker(location=[log,lat],
                      popup = popup, icon=folium.Icon(color='red', icon=''))
        #folium.Marker(location=[lat, log], popup = tag).add_to(m)
        fg4.add_child(Z)
    folium_static(m)
else:
    st.write('Слишком много оценок, поправьте параметры')