"""
Sea Ice Diagnostics.
====================

Diagnostic to produce a series of images which are useful for evaluating
the behaviour of the a sea ice model.

There are three kinds of plots shown here.
1. Sea ice Extent maps plots with a stereoscoic projection.
2. Maps plots of individual models ice fracrtion.
3. Time series plots for the total ice extent.

All three kinds of plots are made for both Summer and Winter in both the
North and Southern hemisphere.

Note that this diagnostic assumes that the preprocessors do the bulk of the
hard work, and that the cube received by this diagnostic (via the settings.yml
and metadata.yml files) has no time component, a small number of depth layers,
and a latitude and longitude coordinates.

This diagnostic takes data from either North or South hemisphere, and
from either December-January-February or June-July-August. This diagnostic
requires the data to be 2D+time, and typically expects the data field to be
the sea ice cover.
An approproate preprocessor would be::

  preprocessors:
    timeseries_NHW_ice_extent: # North Hemisphere Winter ice_extent
      custom_order: true
      extract_time:
          start_year: 1960
          start_month: 12
          start_day: 1
          end_year: 2005
          end_month: 9
          end_day: 31
      extract_season:
        season: DJF
      extract_region:
        start_longitude: -180.
        end_longitude: 180.
        start_latitude: 0.
        end_latitude: 90.

Note that this recipe may not function on machines with no access to the
internet, as cartopy may try to download the shapefiles. The solution to
this issue is the put the relevant cartopy shapefiles on a disk visible to your
machine, then link that path to ESMValTool via the `auxiliary_data_dir`
variable. The cartopy masking files can be downloaded from::

    https://www.naturalearthdata.com/downloads/

Here, cartopy uses the 1:10, physical coastlines and land files::

        110m_coastline.dbf  110m_coastline.shp  110m_coastline.shx
        110m_land.dbf  110m_land.shp  110m_land.shx

This tool is part of the ocean diagnostic tools package in the ESMValTool.

Author: Lee de Mora (PML)
        ledm@pml.ac.uk
"""
import logging
import os
import sys
from itertools import product

import cartopy
import iris
import iris.coord_categorisation
import iris.quickplot as qplt
import matplotlib
import matplotlib.pyplot as plt
import numpy as np

from esmvaltool.diag_scripts.ocean import diagnostic_tools as diagtools
from esmvaltool.diag_scripts.shared import run_diagnostic

# This part sends debug statements to stdout
logger = logging.getLogger(os.path.basename(__file__))
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

# Note that this recipe may not function on machines with no access to
# the internet, as cartopy may try to download geographic files.

def create_ice_cmap(threshold=0.15):
    """
    Create colour map with ocean blue below a threshold and white above.

    Parameters
    ----------
    threshold: float
        The threshold for the line between blue and white.

    Returns
    -------
    matplotlib.colors.LinearSegmentedColormap:
        The resulting colour map.

    """
    threshold = threshold / 100.
    ice_cmap_dict = {
        'red': ((0., 0.0313, 0.0313), (threshold, 0.0313, 1.), (1., 1., 1.)),
        'green': ((0., 0.237, 0.237), (threshold, 0.237, 1.), (1., 1., 1.)),
        'blue': ((0., 0.456, 0.456), (threshold, 0.456, 1.), (1., 1., 1.))
    }

    return matplotlib.colors.LinearSegmentedColormap('ice_cmap', ice_cmap_dict)

def calculate_area_time_series(cube, plot_type, threshold):
    """
    Calculate the area of unmasked cube cells.

    Requires a cube with two spacial dimensions. (no depth coordinate).

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube
    plot_type: str
        The type of plot: ice extent or ice area
    threshold: float
        The threshold for ice fraction (typically 15%)

    Returns
    -------
    numpy array:
        An numpy array containing the time points.
    numpy.array:
        An numpy array containing the total ice extent or total ice area.

    """
    data = []
    times = diagtools.cube_time_to_float(cube)
    for time_itr, time in enumerate(times):
        icedata = cube[time_itr].data

        area = iris.analysis.cartography.area_weights(cube[time_itr])
        if plot_type.lower() == 'ice extent':
            # Ice extend is the area with more than 15% ice cover.
            icedata = np.ma.masked_where(icedata < threshold, icedata)
            total_area = np.ma.masked_where(icedata.mask, area.data).sum()
        if plot_type.lower() == 'ice area':
            # Ice area is cover * cell area
            total_area = np.sum(icedata * area)

        logger.debug('Calculating time series area: %s, %s, %s,', time_itr,
                     time, total_area)
        data.append(total_area)

    ######
    # Create a small dummy output array
    data = np.array(data)
    return times, data

def make_ts_plots(
        cfg,
        metadata,
        filename,
):
    """
    Make a ice extent and ice area time series plot for an individual model.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.
    metadata: dict
        The metadata dictionairy for a specific model.
    filename: str
        The preprocessed model file.

    """
    # Load cube and set up units
    cube = iris.load_cube(filename)
    iris.coord_categorisation.add_year(cube, 'time')
    cube = diagtools.bgc_units(cube, metadata['short_name'])
    cube = agregate_by_season(cube)

    # Is this data is a multi-model dataset?
    multi_model = metadata['dataset'].find('MultiModel') > -1

    # Make a dict of cubes for each layer.
    cubes = diagtools.make_cube_layer_dict(cube)

    # Load image format extention
    image_extention = diagtools.get_image_format(cfg)

    # # Load threshold, pole, season.
    threshold = float(cfg['threshold'])
    pole = get_pole(cube)
    season = get_season(cube)

    # Making plots for each layer
    for plot_type in ['Ice Extent', 'Ice Area']:
        for layer_index, (layer, cube_layer) in enumerate(cubes.items()):
            layer = str(layer)

            times, data = calculate_area_time_series(cube_layer, plot_type,
                                                     threshold)

            plt.plot(times, data)

            # Add title to plot
            title = ' '.join(
                [metadata['dataset'], pole, 'hemisphere', season, plot_type])
            if layer:
                title = ' '.join([
                    title, '(', layer,
                    str(cube_layer.coords('depth')[0].units), ')'
                ])
            plt.title(title)

            # y axis label:
            plt.ylabel(' '.join([plot_type, 'm^2']))

            # Determine image filename:
            suffix = '_'.join(['ts', metadata['preprocessor'], season, pole,
                               plot_type, str(layer_index)])\
                     + image_extention
            suffix = suffix.replace(' ', '')
            if multi_model:
                path = diagtools.folder(
                    cfg['plot_dir']) + os.path.basename(filename)
                path = path.replace('.nc', suffix)
            else:
                path = diagtools.get_image_path(
                    cfg,
                    metadata,
                    suffix=suffix,
                )

            # Saving files:
            if cfg['write_plots']:
                logger.info('Saving plots to %s', path)
                plt.savefig(path)

            plt.close()

def make_polar_map(
        cube,
        pole='North',
        cmap='Blues_r',
):
    """
    Make a polar stereoscopic map plot.

    The cube is the opened cube (two dimensional),
    pole is the polar region (North/South)
    cmap is the colourmap,

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube
    pole: str
        The hemisphere
    cmap: str
        The string describing the matplotlib colourmap.

    Returns
    ----------
    matplotlib.pyplot.figure:
        The matplotlib figure where the map was drawn.
    matplotlib.pyplot.axes:
        The matplotlib axes where the map was drawn.

    """
    fig = plt.figure()
    fig.set_size_inches(7, 7)

    # ####
    # Set  limits, based on https://nedbatchelder.com/blog/200806/pylint.html

    if pole not in ['North', 'South']:
        logger.fatal('make_polar_map: hemisphere not provided.')

    if pole == 'North':  # North Hemisphere
        ax1 = plt.subplot(111, projection=cartopy.crs.NorthPolarStereo())
        ax1.set_extent([-180, 180, 50, 90], cartopy.crs.PlateCarree())

    if pole == 'South':  # South Hemisphere
        ax1 = plt.subplot(111, projection=cartopy.crs.SouthPolarStereo())
        ax1.set_extent([-180, 180, -90, -50], cartopy.crs.PlateCarree())

    linrange = np.linspace(0., 100., 21.)
    qplt.contourf(cube, linrange, cmap=cmap, linewidth=0, rasterized=True)
    plt.tight_layout()

    try:
        ax1.add_feature(
            cartopy.feature.LAND,
            zorder=10,
            facecolor=[0.8, 0.8, 0.8],
        )
    except ConnectionRefusedError:
        logger.error('Cartopy was unable add coastlines due to  a '
                     'connection error.')
    ax1.gridlines(
        linewidth=0.5, color='black', zorder=20, alpha=0.5, linestyle='--')
    try:
        plt.gca().coastlines()
    except AttributeError:
        logger.warning('make_polar_map: Not able to add coastlines')
    return fig

def get_pole(cube):
    """
    Figure out the hemisphere and returns it as a string (North or South).

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The hemisphere (North or South)

    """
    margin = 5.
    if np.max(cube.coord('latitude').points) < 0. + margin:
        return 'South'
    if np.min(cube.coord('latitude').points) > 0. - margin:
        return 'North'
    logger.fatal('get_pole: Not able to determine hemisphere.')
    return False

def get_time_string(cube):
    """
    Return a climatological season string in the format: "year season".

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The climatological season as a string

    """
    season = cube.coord('clim_season').points
    year = cube.coord('year').points
    return str(int(year[0])) + ' ' + season[0].upper()

def get_year(cube):
    """
    Return the cube year as a string.

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The year as a string

    """
    year = cube.coord('year').points
    return str(int(year))

def get_season(cube):
    """
    Return a climatological season time string.

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The climatological season as a string

    """
    season = cube.coord('clim_season').points
    return season[0].upper()

def make_map_plots(
        cfg,
        metadata,
        filename,
):
    """
    Make a simple map plot for an individual model.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.
    metadata: dict
        The metadata dictionairy for a specific model.
    filename: str
        The preprocessed model file.

    """
    # Load cube and set up units
    cube = iris.load_cube(filename)
    iris.coord_categorisation.add_year(cube, 'time')
    cube = diagtools.bgc_units(cube, metadata['short_name'])
    cube = agregate_by_season(cube)

    # Is this data is a multi-model dataset?
    multi_model = metadata['dataset'].find('MultiModel') > -1

    # Make a dict of cubes for each layer.
    cubes = diagtools.make_cube_layer_dict(cube)

    # Load image format extention and threshold.
    image_extention = diagtools.get_image_format(cfg)
    threshold = float(cfg['threshold'])

    # Making plots for each layer
    plot_types = ['Fractional cover', 'Ice Extent']
    plot_times = [0, -1]
    for plot_type, plot_time in product(plot_types, plot_times):
        for layer_index, (layer, cube_layer) in enumerate(cubes.items()):
            layer = str(layer)

            if plot_type == 'Fractional cover':
                cmap = 'Blues_r'
            if plot_type == 'Ice Extent':
                cmap = create_ice_cmap(threshold)

            cube = cube_layer[plot_time]

            # use cube to determine which hemisphere, season and year.
            pole = get_pole(cube)
            time_str = get_time_string(cube)

            # Make the polar map.
            make_polar_map(cube, pole=pole, cmap=cmap)

            # Add title to plot
            title = ' '.join([metadata['dataset'], plot_type, time_str])
            if layer:
                title = ' '.join([
                    title, '(', layer,
                    str(cube_layer.coords('depth')[0].units), ')'
                ])
            plt.title(title)

            # Determine image filename:
            suffix = '_'.join(
                ['ortho_map', plot_type, time_str,
                 str(layer_index)])
            suffix = suffix.replace(' ', '') + image_extention
            if multi_model:
                path = diagtools.folder(cfg['plot_dir'])
                path = path + os.path.basename(filename)
                path = path.replace('.nc', suffix)
            else:
                path = diagtools.get_image_path(
                    cfg,
                    metadata,
                    suffix=suffix,
                )

            # Saving files:
            if cfg['write_plots']:
                logger.info('Saving plots to %s', path)
                plt.savefig(path)

            plt.close()

def agregate_by_season(cube):
    """
    Aggregate the cube into seasonal means.

    Note that it is not currently possible to do this in the preprocessor,
    as the seasonal mean changes the cube units.

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    iris.cube.Cube:
        Data Cube with the seasonal means

    """
    if not cube.coords('clim_season'):
        iris.coord_categorisation.add_season(cube, 'time', name='clim_season')
    if not cube.coords('season_year'):
        iris.coord_categorisation.add_season_year(
            cube, 'time', name='season_year')
    return cube.aggregated_by(['clim_season', 'season_year'],
                              iris.analysis.MEAN)

def make_map_extent_plots(
        cfg,
        metadata,
        filename,
):
    """
    Make an extent map plot showing several times for an individual model.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.
    metadata: dict
        The metadata dictionairy for a specific model.
    filename: str
        The preprocessed model file.

    """
    # Load cube and set up units
    cube = iris.load_cube(filename)
    iris.coord_categorisation.add_year(cube, 'time')
    cube = diagtools.bgc_units(cube, metadata['short_name'])
    cube = agregate_by_season(cube)

    # Is this data is a multi-model dataset?
    multi_model = metadata['dataset'].find('MultiModel') > -1

    # Make a dict of cubes for each layer.
    cubes = diagtools.make_cube_layer_dict(cube)

    # Load image format extention
    image_extention = diagtools.get_image_format(cfg)

    # Load threshold, pole and season
    threshold = float(cfg['threshold'])
    pole = get_pole(cube)
    season = get_season(cube)

    # Start making figure
    for layer_index, (layer, cube_layer) in enumerate(cubes.items()):

        fig = plt.figure()
        fig.set_size_inches(7, 7)

        if pole == 'North':  # North Hemisphere
            projection = cartopy.crs.NorthPolarStereo()
            ax1 = plt.subplot(111, projection=projection)
            ax1.set_extent([-180, 180, 50, 90], cartopy.crs.PlateCarree())

        if pole == 'South':  # South Hemisphere
            projection = cartopy.crs.SouthPolarStereo()
            ax1 = plt.subplot(111, projection=projection)
            ax1.set_extent([-180, 180, -90, -50], cartopy.crs.PlateCarree())
        try:
            ax1.add_feature(
                cartopy.feature.LAND, zorder=10, facecolor=[0.8, 0.8, 0.8])
        except ConnectionRefusedError:
            logger.error('Cartopy was unable add coastlines due to  a '
                         'connection error.')

        ax1.gridlines(
            linewidth=0.5, color='black', zorder=20, alpha=0.5, linestyle='--')

        try:
            plt.gca().coastlines()
        except AttributeError:
            logger.warning('make_polar_map: Not able to add coastlines')

        times = np.array(cube.coord('time').points.astype(float))
        plot_desc = {}
        for time_itr, time in enumerate(times):
            cube = cube_layer[time_itr]
            line_width = 1
            color = plt.cm.jet(float(time_itr) / float(len(times)))
            label = get_year(cube)
            plot_desc[time] = {'label': label,
                               'c': [color, ],
                               'lw': [line_width, ],
                               'ls': ['-', ]}

            layer = str(layer)
            qplt.contour(cube,
                         [threshold, ],
                         colors=plot_desc[time]['c'],
                         linewidths=plot_desc[time]['lw'],
                         linestyles=plot_desc[time]['ls'],
                         rasterized=True)

        # Add legend
        legend_size = len(plot_desc) + 1
        ncols = int(legend_size / 25) + 1
        ax1.set_position([
            ax1.get_position().x0,
            ax1.get_position().y0,
            ax1.get_position().width * (1. - 0.1 * ncols),
            ax1.get_position().height
        ])

        fig.set_size_inches(7 + ncols * 1.2, 7)

        # Construct dummy plots.
        for i in sorted(plot_desc):
            plt.plot(
                [],
                [],
                c=plot_desc[i]['c'][0],
                lw=plot_desc[i]['lw'][0],
                ls=plot_desc[i]['ls'][0],
                label=plot_desc[i]['label'],
            )

        legd = ax1.legend(
            loc='center left',
            ncol=ncols,
            prop={'size': 10},
            bbox_to_anchor=(1., 0.5))
        legd.draw_frame(False)
        legd.get_frame().set_alpha(0.)

        # Add title to plot
        title = ' '.join([
            metadata['dataset'],
        ])
        if layer:
            title = ' '.join([
                title, '(', layer,
                str(cube_layer.coords('depth')[0].units), ')'
            ])
        plt.title(title)

        # Determine image filename:
        suffix = '_'.join(['ortho_map', pole, season, str(layer_index)])
        suffix = suffix.replace(' ', '') + image_extention
        if multi_model:
            path = diagtools.folder(cfg['plot_dir'])
            path = path + os.path.basename(filename)
            path = path.replace('.nc', suffix)
        else:
            path = diagtools.get_image_path(
                cfg,
                metadata,
                suffix=suffix,
            )

        # Saving files:
        if cfg['write_plots']:
            logger.info('Saving plots to %s', path)
            plt.savefig(path)
        plt.close()

def main(cfg):
    """
    Load the config file and metadata, then pass them the plot making tools.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.

    """
    cartopy.config['data_dir'] = cfg['auxiliary_data_dir']

    for index, metadata_filename in enumerate(cfg['input_files']):
        logger.info(
            'metadata filename:\t%s',
            metadata_filename,
        )

        metadatas = diagtools.get_input_files(cfg, index=index)
        for filename in sorted(metadatas):

            logger.info('-----------------')
            logger.info(
                'model filenames:\t%s',
                filename,
            )
            ######
            # extent maps plots of individual models
            make_map_extent_plots(cfg, metadatas[filename], filename)

            ######
            # maps plots of individual models
            make_map_plots(cfg, metadatas[filename], filename)

            ######
            # time series plots o
            make_ts_plots(cfg, metadatas[filename], filename)

    logger.info('Success')

if __name__ == '__main__':
    with run_diagnostic() as config:
        main(config)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""

@author: Frank Nussbaum (frank.nussbaum@uni-jena.de), 2019

"""
import abc
import numpy as np

#import time

# pylint: disable=R0914

class BaseAdmm(abc.ABC):
    """A base class for ADMM solvers.

    Attributes:
        admm_param (float): initial admm parameter
        opts (dict): dictionary of solver options
    """

    def __init__(self):
        super().__init__()
        self.admm_param = 1  # TODO(franknu): external access
        self.opts = {}
        self.set_defaults_admm()

    @abc.abstractmethod
    def _do_iter_admm(self, current_vars: tuple):
        """perform computations of one ADMM iteration"""
        raise NotImplementedError  # override in derived classes

    @abc.abstractmethod
    def _initialize_admm(self):
        """initialize ADMM variables"""
        raise NotImplementedError  # override in derived classes

#    def _collect(self, current_vars: tuple):
#        """store information of ADMM variables in dictionairy"""
#        raise NotImplementedError  # override in derived classes

    def _report(self, out):
        """print stats after solving"""
        print('>', out['message'])
        print('> (P)ADMM: obj: %e, iter: %d, resid:%e' %
              (out['obj'], out['iter'], out['resid']))
        try:
            print('> regparams used:', self.get_regparams())
        except AttributeError:
            # Method get_regparams does not exist
            pass

        for i, var in enumerate(out['solution']):
            print('norm optvar%d=%f' % (i, np.linalg.norm(var)))

    def set_defaults_admm(self):
        """Set default solver options in class attribute opt.

        Adjustable solver options contain abstol, reltol,
        maxiter, stoptol, verb, etc."""
        self.opts = {}
        self.opts.setdefault('abstol', 1e-5)
        self.opts.setdefault('reltol', 1e-5)

        self.opts.setdefault('continuation', 1)
        self.opts.setdefault('num_continuation', 10)
        #        self.opts.setdefault('cont_adaptive', 10)
        self.opts.setdefault('continuation_fac', 2)
        #        self.opts.setdefault('eta', .25)
        #        self.opts.setdefault('muf', 1e-6)
        self.opts.setdefault('maxiter', 500)
        self.opts.setdefault('stoptol', 1e-5)

        self.opts.setdefault('verb', 0)

    def _solve(self, warminit=None):
        """
        solve a generic ADMM problem
        """
        ## initialization
        current_vars = self._initialize_admm()
        if not warminit is None:
            current_vars = warminit

        hist = np.empty((5, self.opts['maxiter'] + 1))
        history = {
            'objval': hist[0, :],
            'r_norm': hist[1, :],
            's_norm': hist[2, :],
            'eps_pri': hist[3, :],
            'eps_dual': hist[4, :]
        }

        for i in range(1, self.opts['maxiter'] + 1):
            ## update Theta
            current_vars, residuals, stats = self._do_iter_admm(current_vars)

            ## diagnostics, reporting, termination checks
            rnorm, snorm, eps_pri, eps_dual = residuals

            history['objval'][i] = stats['admm_obj']

            history['r_norm'][i] = rnorm
            history['s_norm'][i] = snorm

            history['eps_pri'][i] = eps_pri
            history['eps_dual'][i] = eps_dual

#            print(i % self.opts['verb'])
            if self.opts['verb'] and i % self.opts['verb'] == 0:
                print('%3d\t%10.4f %10.4f %10.4f %10.4f %10.2f' %
                      (i, history['r_norm'][i], history['eps_pri'][i],
                       history['s_norm'][i], history['eps_dual'][i],
                       history['objval'][i]))  # resid %10.2f

            ## check stop
            pridualresids_below_tolerance = (rnorm < eps_pri and
                                             snorm < eps_dual)

            if pridualresids_below_tolerance:
                break

            if self.opts['continuation'] and \
                i % self.opts['num_continuation'] == 0:
                # self-adaptive update
                # generally: larger admm_param leads to lower penalty on
                # primal residual (resid)
                admm_param_old = self.admm_param

                admm_state = eps_pri, eps_dual, rnorm, snorm

                if (i / self.opts['num_continuation']) % 5 == 1:
                    self._cont_update_2019(admm_state)
                else:
                    self._cont_update_s2a(admm_state)

                if self.opts['verb'] and self.admm_param != admm_param_old:
                    print('>> New ADMM parameter', self.admm_param,
                          '(old was %f)' % (admm_param_old))

        out = {}
        if pridualresids_below_tolerance:
            out['message'] = b'CONVERGENCE: primal and dual residual below tolerance'
        else:
            out['message'] = b'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'

        out['history'] = history
        out['iter'] = i
        # out['admm_obj'] = history['objval'][i] # already in stats, see below

        out.update(stats)

        return out

    def _cont_update_s2a(self, admm_state, criticalratio: int = 5):
        """ update S2a from my master thesis
        uses more robust constant size updates of admm_param"""

        eps_pri, eps_dual, rnorm, snorm = admm_state

        snorm_rel = snorm / eps_dual
        rnorm_rel = rnorm / eps_pri
        if snorm_rel < 1 and rnorm_rel < 1:
            return  # residuals are already smaller than tolerancies

#        print(snorm_rel, rnorm_rel)

        ## do scaling of admm_param if necessary
        if rnorm_rel > snorm_rel * criticalratio:
            # decrease admm_param
            self.admm_param /= self.opts['continuation_fac']
        elif snorm_rel > rnorm_rel * criticalratio:
            # increase admm_param
            self.admm_param *= self.opts['continuation_fac']

    def _cont_update_2019(self,
                         admm_state,
                         criticalratio: int = 5,
                         cutoff: tuple = (1e-2, 1e2)):
        """ update that respects current ratio of residuals with tolerancies"""
        # TODO(franknu): consider speed of improvement instead of
        # just current state? - hist as class variable??
        # ML to learn update?
        eps_pri, eps_dual, rnorm, snorm = admm_state
        snorm_rel = snorm / eps_dual
        rnorm_rel = rnorm / eps_pri

        if snorm_rel < 1 and rnorm_rel < 1:
            return  # residuals are already smaller than tolerancies

        # do cutoff
        lower, upper = cutoff
        snorm_rel = min(upper, max((snorm_rel, lower)))
        rnorm_rel = min(upper, max((rnorm_rel, lower)))

        if snorm_rel < 1:
            # decrease ADMM param
            # increases penalty for violations of primal feasibiity
            scaling_factor = 1 / rnorm_rel
        elif rnorm_rel < 1:
            # increase ADMM param
            # decreases penalty for violations of primal feasibiity
            scaling_factor = snorm_rel
        else:
            scaling_factor = snorm_rel / rnorm_rel

#        if self.opts['verb']:
#            print(snorm_rel, rnorm_rel)

        ## do scaling of admm_param if necessary
        if max((scaling_factor, 1 / scaling_factor)) > criticalratio:

            if rnorm_rel >= 1 and snorm_rel >= 1:
                scaling_factor = (scaling_factor)**.8
                # inhibition, draw towards 1

            if self.opts['verb']:
                print('scaling_factor', scaling_factor)
            self.admm_param *= scaling_factor

    def solve(self, report=False, warminit=None, **kwargs):
        """solve the problem

        Args:
            report (bool): whether to print additional summary.
            warminit (solver state): starting point for the solver.
            kwargs (optional): additional solver options to replace defaults.

        Returns:
            dict: dictionary with info about solution/solving process
        """

        self.opts.update(**kwargs) # update solver options

        if report:
            print('>solver options', self.opts)

        ## select appropriate sparsity-inducing norms and corresponding proximity operators

        out = self._solve(warminit=warminit)

        if report:
            self._report(out)

        if not out['message'].startswith(b'CONV'):
            print(out['message'])

        return out


# -*- coding: utf-8 -*-
"""
Created on Sun May 10 09:59:15 2020

@author: prophet lin
"""

import os
import pandas as pd
import numpy as np
out = open("./res.txt", 'w+')

#%% 读取生成式  @表示为空
generator = {}
start = None    #开始符号
ter_set = set() #终结符集合
non_set = set() #非终结符集合
all_set = set()
#grammar2
f = open('./tiny.txt')
for line in f:
    line = line.split('->')
    #添加开始符号
    if not non_set:
        start = line[0].split()[0]
    #非终结符
    non_sym = line[0].split()[0]
    non_set.add(non_sym)
    #右侧所有符号
    all_syms = line[1].split() #[:-1] delete '/n'
    all_set = all_set.union(set(all_syms))

    if non_sym not in generator:
        generator[non_sym] = []
    generator[non_sym].append(all_syms)
#求非终结符集
ter_set = all_set - non_set
print("起始符号：{0}\n非终结符：{1}\n终结符：{2}\n===========".format(start, non_set, ter_set), file=out)
#%% 消除左递归

#%% 提取左因子

#%% 生成First集
first = {}
#第一轮 把所有开头的非终结符和空放入First集
for l_sym in generator:
    for each in generator[l_sym]:
        if l_sym not in first:
            first[l_sym] = set()
        if each[0] in ter_set:
            first[l_sym].add(each[0])
#循环轮 添加非终结符，循环直到不再变化
update = 1
while update == 1:
    update = 0
    for l_sym in generator:
        for each in generator[l_sym]:
            #处理空符号，尤其是前面所有的均为空
            emputy_flag = 0
            for i in range(len(each)):
                if each[i] in non_set:
                    #如果添加导致改变，设置标志位
                    temp = first[l_sym].union( first[each[i]] - set('@') )
                    if first[l_sym] != temp:
                        update = 1
                        first[l_sym] = temp
                if each[i] in ter_set and each[i] not in first[l_sym]:
                    update = 1
                    first[l_sym].add(each[i])
                if each[i] in ter_set or (each[i] in non_set and '@' not in first[each[i]]):
                    emputy_flag = 1
                    break

            if emputy_flag == 0:
                first[l_sym].add('@')

print("First集：", file=out)
for item in first:
    print("{0}:{1}".format(item,first[item]), file=out)
print("===========", file=out)
#%% 生成Follow集
# $ 加入起始符号
follow = {}
for non_sym in non_set:
    follow[non_sym] = set()
follow[start] = set('$')

# 第一轮 将非终结符后的终结符添加到FOLLOW中
for l_sym in generator:
    for each in generator[l_sym]:
        for i in range(len(each)-1):
            if each[i] in non_set and each[i+1] in ter_set:
                follow[each[i]].add(each[i+1])
#第二轮 集合间的同步，非终结符后的非终结符first集，左侧添加到右侧最后，循环直到不再变化
update = 1
while update == 1:
    update = 0
    for l_sym in generator:
        for each in generator[l_sym]:
            # 产生式后加入$,用来合并逻辑,不必从末尾向前找空，而是顺序向后
            temp_produce = each.copy()
            temp_produce.append('$')
            for i in range(len(each)):
                next_p = i
                while True:
                    next_p += 1
                    #非终结符后的非终结符first集
                    if temp_produce[i] in non_set and temp_produce[next_p] in non_set:
                        temp = follow[temp_produce[i]].union( first[temp_produce[next_p]] - set('@') )
                        if follow[temp_produce[i]] != temp:
                            update = 1
                            follow[temp_produce[i]] = temp
                    #左侧follow添加到右侧最后
                    if temp_produce[i] in non_set and temp_produce[next_p] == '$':
                        temp = follow[temp_produce[i]].union( follow[l_sym] )
                        if follow[temp_produce[i]] != temp:
                            update = 1
                            follow[temp_produce[i]] = temp
                    if temp_produce[next_p] in ter_set or temp_produce[next_p] == '$' or ( '@' not in first[temp_produce[next_p]]):
                        break
print("Follow集：", file=out)
for item in follow:
    print("{0}:{1}".format(item,follow[item]), file=out)
print("===========", file=out)
#%% 生成LL1分析表 判断是否是LL1文法
vaild = True
ter_list = list(ter_set)
non_list = list(non_set)
ter_list.append('$')
ter_list.remove('@')
ll1_table = pd.DataFrame(columns = ter_list,index = non_list)
generator_list = []
count = 0
for l_sym in generator:
    for each in generator[l_sym]:
        #print(each,count)
        generator_list.append([l_sym,each])
        if each[0] == '@':
            #添加Follow
            for sym in follow[l_sym]:
                if np.isnan(ll1_table.loc[l_sym,sym]):
                    ll1_table.loc[l_sym,sym] = count
                else:
                    vaild = False
            count += 1
            continue
        if each[0] in ter_set:
            if np.isnan(ll1_table.loc[l_sym,each[0]]):
                ll1_table.loc[l_sym,each[0]] = count
            else:
                vaild = False
        #TODO:处理first中有空的情况
        if each[0] in non_set:
            for sym in first[each[0]]:
                if np.isnan(ll1_table.loc[l_sym,sym]):
                    ll1_table.loc[l_sym,sym] = count
                else:
                    vaild = False
        count += 1

if not vaild:
    print('文法不满足LL（1）条件！')
ll1_table.to_excel('LL1.xlsx')
#%% 分析栈分析代码 以及构造抽象语法树
token_file = open('./token_list.txt')
tokens = []
for line in token_file:
    tokens.append(line[:-1])
tokens.append('$')
stack = []
stack.append("$")
stack.append(start)
while stack:
    print('=====================\n','stack:',stack,'\n tokens:',tokens, file=out)

    #match
    if stack[-1] in ter_set or (stack[-1] == '$' and tokens[-1] == '$'):
        if stack[-1] == tokens[0]:
            stack.pop()
            tokens.pop(0)
            continue
        else:
            print('不满足文法')
            break
    #action
    if stack[-1] in non_set:
        if np.isnan(ll1_table.loc[stack[-1],tokens[0]]):
            print('不满足文法')
            break
        else:
            use_p = generator_list[ll1_table.loc[stack[-1],tokens[0]]]
            stack.pop()
            temp = use_p[1].copy()
            if temp[0] != '@':
                temp.reverse()
                stack.extend(temp)

#%%
out.close()
'''
update = 1
while update == 1:
    update = 0
    for l_sym in generator:
        #TODO:处理空符号，尤其是前面所有的均为空
        for each in generator[l_sym]:
            if each[0] in non_set:
                #如果添加导致改变，设置标志位
                temp = first[l_sym].union( first[each[0]] - set('@') )
                if first[l_sym] != temp:
                    update = 1
                    first[l_sym] = temp
'''


#!/usr/bin/env python
# coding: utf-8

# Importing the required libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import streamlit as st
import streamlit.components.v1 as components
from PIL import Image
import plotly.express as px
from streamlit_timeline import timeline
import plotly.graph_objects as go
import seaborn as sns
import plotly.figure_factory as ff

###########################

st.set_page_config(layout="wide")

st.title('How various factors have influenced the housing prices over the past 2 decades - A case Study 📔')

st.sidebar.markdown('''
# Contents
- [Introduction](#introduction)
- [Brief Timeline](#a-brief-timeline-of-major-economic-events)
- [The Past 2 Decades](#factors-and-how-they-influenced-the-past-2-decades)
    - [Demand](#demand)
        - [Population](#population)
        - [Unemployment](#unemployment)
        - [Income](#income)
        - [Mortgage Rate](#mortgage-rate)
        - [Debt](#household-debt)
    - [Supply](#supply)
        - [Permits](#new-single-family-unit-permits)
        - [Existing home sales](#existing-home-sales)
        - [Foreclosures](#foreclosures)
- [Most Influencial Factor](#determining-the-most-influencial-factor)
    - [Correlation Analysis](#correlation-analysis)
    - [Explanatory Regression Analysis](#explanatory-regression-analysis)
- [Conclusion](#conclusion)
- [Biblography](#biblography)

''', unsafe_allow_html=True)

###########################

st.header('Introduction')
col1,col2 = st.columns((2,1))
col1.write('''

Saying that the housing market is influenced by a myriad of factors could be an understatement.
The entire process of reverse engineering something as complex and as complicated as a housing market requires in-depth and
unbiased analysis which in turn produces credible predictive analytics.

The **S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index** measures the change in the value of the U.S.
residential housing market by tracking the purchase prices of single-family homes, which I have used as a proxy for home prices.

As of October 2021, there were 208.4 million single-family dwelling units in the United States and only 37.8 million multifamily units.
**The size of single-family housing units has steadily risen since the start of the 21st century.**

There is an upward trend in the average size of floor area in new single-family homes. In 1975, a new home had an average floor area of about 1,660 square feet.
By 2019, this figure had **increased by** about 900 square feet.

''')

labels = ['Multi-family dwelling unit', 'Single Family dewlling unit', 'Others']
sizes = [37.75, 208.36, 10.67]
explode = (0, 0.1, 0)  # only "explode" the 2nd slice

fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90)
ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
ax1.set_title("The single-family units as a fraction of all the housing units in USA", pad = 20)

col2.pyplot(fig1)

st.write('''
The interavtive rgraph below shows how the S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index has changed in the past 2 years.
The data has been adjusted to show quaterly mean.
''')

col1,col2 = st.columns((2,1))

df = pd.read_csv('./data/cleaned/snp.csv')
df.rename(columns = {'CSUSHPISA':'Index'}, inplace = True)
fig = px.line(df, x='DATE', y='Index')
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
col1.plotly_chart(fig)

col2.markdown('''_
\n
The Case Shiller Home Price Index averaged 169.87 points from 2000 until 2021.
\n It reached an all-time high of 257.10 points in April 2021.
\nIt reached a record low of 100 points in January 2000.
''')

###########################
st.header('A Brief Timeline ⏱ of Major Economic Events')
with open('data.json', "r") as f:
    data = f.read()

# render timeline
timeline(data, height=800)

#####################
st.header('Factors and How They Influenced The Past 2 Decades 📊')
st.header('Population 🧑🏻🤝🧑🏾')
colm1,colm2 = st.columns((3,4))
colm1.write('''
The U.S. population increased by 10 percent between 2000 and 2010 and is projected to increase by 8 percent between 2010 and 2020, from 309 million to 333 million. An 8 percent gain would be the smallest percentage increase in the U.S. population between censuses since the 1930s.
\nThe population-housing relationship is a two-way street. On the one hand, population growth causes a shift in housing demand. Population growth, and especially an increase in the number of households, leads to an increase in housing demand.
\nPopulation decline might lead to a decrease in housing demand. This will, however, only happen in the long run, after not only the number of people but also the number of households has started to decline. The danger of population decline is greatest in remote rural areas and in areas with lower-quality housing.
At the same time, the supply of housing influences the opportunities for population increase through migration.

''')

df = pd.read_csv('./data/cleaned/population_q_cleaned.csv')
df.rename(columns = {'POPTHM':'Population'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

col1,col2 = st.columns((3,4))
df = pd.read_csv('./data/cleaned/pops.csv')
df.rename(columns = {'CNP16OV_TTLHH':'No. of members in a household'}, inplace = True)
fig = px.line(df, x='Date', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
with col2:
    st.plotly_chart(fig)

with col1:
    st.write('**Poplulation level/Total household** 🧑🏻🤝🧑🏾/🏠')
    st.write('''
    The graph shows that the number of people in each household has decreased in the longer time period. So the number of households in the US has increased faster than population, which means that any measure divided by population grows faster than one divided by number of households.
    ''')

#####################
st.header('Unemployment')
colm1,colm2 = st.columns((3,4))
df = pd.read_csv('./data/cleaned/unemp.csv')
df.rename(columns = {'UNRATE':'Unemployment Rate in %'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

with colm1:
    components.iframe("https://d3fy651gv2fhd3.cloudfront.net/embed/?s=usurtot&v=202203041337V20200908&d1=19970314&type=type=line&title=false&url2=/united-states/case-shiller-home-price-index&h=300&w=600",width=600, height=300, scrolling=False)
    st.write("Unemployment tracks the business cycle. Recessions are part of that cycle and can cause high unemployment. Businesses often lay off workers and, without an income, those jobless workers have less money to spend.")

st.write('''
Lower consumer spending reduces business revenue, which forces companies to cut more payroll. This downward cycle can be devastating to individuals and the economy.
When unemployment is rising, fewer people will be able to afford a house. But, even the fear of unemployment may discourage people from entering the property market.
An increase in jobs, employees and wages consequently results in an increase in income for all workers in a country. With that, an increase in income also increases the demand for affordable housing.
\nDuring the Great Recession, unemployment reached 10% in October 2009.
In 2020, it reached double digits again at 14.7% in April when the U.S. was dealing with a pandemic and recession.
The US unemployment rate edged down to 3.8 percent in February of 2022 from 4 percent in the previous month, a new pandemic low and below market expectations of 3.9 percent. The number of unemployed persons edged down by 243 thousand to 6.270 million.

''')
############################
st.header('Income 💵')
colm1,colm2 = st.columns((3,4))
colm1.write('''
The shortfall in household income is attributable in part to two recessions since 2000. The first recession, lasting from March 2001 to November 2001, was relatively short-lived.7 Yet household incomes were slow to recover from the 2001 recession and it was not until 2007 that the median income was restored to about its level in 2000.

\nBut 2007 also marked the onset of the Great Recession, and that delivered another blow to household incomes. This time it took until 2015 for incomes to approach their pre-recession level. Indeed, the median household income in 2015 – $70,200 – was no higher than its level in 2000, marking a 15-year period of stagnation, an episode of unprecedented duration in the past five decades.8

\nMore recent trends in household income suggest that the effects of the Great Recession may finally be in the past. From 2015 to 2018, the median U.S. household income increased from 70,200 to 74,600 dollars, at an annual average rate of 2.1%.
''')

col1,col2 = st.columns((3,4))
with col2:
    img = Image.open("aff.png")
    st.image(img)

with col1:
    st.write('**Affordability of Homes**')
    st.write('''

Steady job growth and an increase in the median wage continue to fuel housing demand. Aside from rising wages, another good sign of financial health is spending less disposable income on repaying debts like credit cards, auto loans, and personal loans.

Today, household budgets continue to be the strongest they’ve been in over 40 years and a far cry from 2008, suggesting consumers have more disposable income to buy a home.

    ''')

df = pd.read_csv('./data/cleaned/incomenew.csv')
df.rename(columns = {'MEHOINUSA672N':'Median Income'}, inplace = True)
df.rename(columns = {'HOUSTCB1FQ_HOUSTOB1FQ':'New Privately Owned Homes'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
###########################

st.header('Mortgage Rate')
colm1,colm2 = st.columns((3,4))
colm1.write('''
Monetary policy affects interest rates, which affect mortgages, which affect housing market decisions. That may be simple to grasp, but the housing statistics may not follow such clear-cut patterns.

\nThe graph's red line represents the average 30-year fixed-rate mortgage for the last 20 years. For the same time period, the blue line in the graph represents the ratio of house starts built by contractors to housing starts built by owners.

\nFrom 2000 to 2007, this ratio remained largely steady, hovering around 1.5, meaning that contractors built roughly 60% of new starts. However, during periods of macroeconomic turmoil, the ratio has deviated from its historical average.
\nBut during the Great Recession, this ratio increased sharply, to over 2.0, peaking at 2.6 in 2016, which implies contractors built 72% of housing starts. In both cases, GDP declined and unemployment rose, but this housing measure behaved differently.
''')

st.write('''
A clear difference between these two episodes is the level of mortgage rates: Rates were much higher in the Great Recession. As mortgage rates go up, the ratio goes down and vice versa.
A potential reason is that, as the price of mortgages increases, the cost of purchasing a new home from a contractor increases relative to the cost of building one’s own home. And, if the costs are basically the same,
many would-be homeowners might choose to build their own home rather than purchase one that someone else built.
''')

df = pd.read_csv('./data/cleaned/dd.csv')
df.rename(columns = {'MORTGAGE30US':'Mortgage Rate'}, inplace = True)
df.rename(columns = {'HOUSTCB1FQ_HOUSTOB1FQ':'New Privately Owned Homes'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

#####################

st.header('Household Debt')
colm1,colm2 = st.columns((3,4))
colm1.write('''
There are many types of debt, including household debt, and many specific types of household debt as well.
\nThe graph on the right shows Mortgage Debt Service Payments as a Percent of Disposable Personal Income.
\nThe financial burdens from mortgages debt vary quite a bit. Let’s consider two reasons for this: The larger
the debt, the larger the burden, as households need to pay more interest on a larger principal. And changes
in interest rates obviously influence how much is paid to service loans.\n The mortgage debt increased from the early 1990s until the past recession,
when it decreased. This decrease is the result of the combination of the two effects noted above: the amount of debt and interest rates.
With one exception (in the fourth quarter of 2012), total debt obligations are at the lowest they’ve been since these data were first
collected. And this is especially true of mortgage debt.

''')

df = pd.read_csv('./data/cleaned/debt_inc.csv')
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

labels = ['Mortgage','Student','Others']
values = [10.44, 1.58, 3.22]
# Use `hole` to create a donut-like pie chart
fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.3)])

col1,col2 = st.columns((1,2))
with col2:
    st.plotly_chart(fig)

with col1:
    st.write('**Mortgage debt as a fraction of total household debt**')
    st.write('''
    Mortgage Debt Service Payments as a Percent of Disposable Personal Income was 3.81% in April of 2021, according to the United States Federal Reserve.\nHistorically, United States - Mortgage Debt Service Payments as a Percent of Disposable Personal Income reached a record high of 7.18 in October of 2007 and a record low of 3.51 in January of 2021.
    By the end of 2021, a record level of household debt was reached. Mortgage being the biggest contributor to it being 68.5% at 10.44 Trillion Dollars. Student debt was the second highest contributor with 21.1% at 1.58 Trillion Dollars.

    ''')

##############################

st.header('New Single Family Unit Permits 📋')
colm1,colm2 = st.columns((3,4))
colm1.write('''
\nSingle-family housing starts were pridicted to total 1 million in 2020, the highest since 2007. However,
There were only approximately 978,000 building permits for single-family housing units granted in the United States in 2020, which is
an increase of around 116,000 on the previous year. Units have steadily increased since 2011, but the numbers remain considerably
lower than the 1.68 million authorized by building permits in 2005.
\nThat was about 3 years prior to the housing market meltdown that spurred a global financial rout.
\nThe rising cost of land, labor, building materials, and regulations, are some of the reasons why we can see significantly more blue (people who may need homes) in the graph below, than orange (total homes built) since 2008. You can also see that few of the new homes built were “starter homes” focused on first-time buyers.
''')

df = pd.read_csv('./data/cleaned/permit.csv')
df.rename(columns = {'PERMIT1':'No. of permits issued in thousands'}, inplace = True)
df.rename(columns = {'HSN1F':'No. of one family homes sold in thousands'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
col1,col2 = st.columns((1,1))
img = Image.open('new_perms.png')
with col1:
    st.image(img)

with col2:
    st.write('''
    \n Even though the new homes built by 2020 were the fastest rate since 2007, Single-family home construction is running at the slowest pace since 1995.
    \nThe U.S. is short 5.24 million homes, an increase of 1.4 million from the 2019 gap of 3.84 million.
    \nSingle-family home construction has suffered from a severe labor shortage that began well before the pandemic but was then exacerbated by it. Supply chain disruptions in the past year have pushed prices for building materials higher, and as pandemic-induced demand soared, prices for land increased as well. While new household formation is actually slower than it was before the pandemic, homebuilders would have to double their recent new home production pace to close the gap in five to six years. A new household can be either owner-occupied or rented.
    ''')

################################

st.header('Existing Home Sales')
colm1,colm2 = st.columns((3,4))
colm1.write('''
The steady rise in sales after the sharp drop in 2008 is indicative of the general consensus that the housing market is recovering. Construction is showing positive signs, consumers are growing in confidence and are becoming freer with their spending and the market is entering new periods of growth.
\n
This is a far cry from the dire situation in the not too distant past, in the run up to the bursting of the U.S. housing bubble. Interest rates were very low at that time, making credit cheap and abundantly available. Banks and lending institutions led people to believe that it was okay to buy multiple properties with little money and that real estate was just about the safest investment anyone could make. More and more people decided to take the risk and invest in the market. This coupled with the increased number of people descending on the market caused prices to soar; it seemed like an easy way to make cash fast. But this is how the bubble formed and it was this bubble, upon bursting, that would set into motion a chain of events that would bring the global economy to its knees; plunging the world into an economic depression of which it has not seen the likes since the Great Depression of the 1930s.
''')

df = pd.read_csv('./data/cleaned/existinghomessold.csv')
fig = px.line(df, x='Year', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
col1,col2 = st.columns((2,1))

labels = ['Exisiting Homes Sold','New Homes Sold']
values = [5640000, 822000]

fig = go.Figure(data=[go.Pie(labels=labels, values=values, textinfo='label+percent',
                             insidetextorientation='radial'
                            )])
col1.plotly_chart(fig)

with col2:
    st.write('''
    **U.S. existing home sales 2005-2023**
    \nIn 2021, the U.S. home sales in the United States surged, reaching the highest value observed since 2006. A total of 6.1 million housing transactions were completed in that year, up from 5.6 million in 2020. According to the forecast, sales activity is expected to slow down slightly in 2022 and increase again in 2023.
    ''')

##################################

st.header('Foreclosures 🏦')
colm1,colm2 = st.columns((3,4))
colm1.write('''
Despite a deep recession, the U.S. housing market in 2020 set a record for the fewest foreclosures ever.

Just 214,323 properties were in some stage of foreclosure last year, well below the previous low set in 2019, according to ATTOM Data Solutions. Properties with foreclosure filings in 2020 represented a scant 0.16 percent of all U.S. homes, down from a foreclosure rate of 0.36 percent in 2019. The record high was 2.23 percent, in 2010.

In a separate study released Friday, mortgage data firm Black Knight said foreclosure starts plunged by 67 percent from 2019, while foreclosure sales dropped by 70 percent compared to the previous year.

However, the data isn’t all rosy. Fully 2.15 million American homeowners were 90 days past due on their mortgages, a number that rose by 1.7 million during the year of the coronavirus pandemic, Black Knight said.

Foreclosures fell not because all was well economically, but because lenders essentially stopped taking back properties in 2020. After the coronavirus pandemic struck in March, federal and state officials hit the pause button on default filings by lenders. And the CARES Act called for mortgage forbearance plans designed to keep struggling workers in their homes.
''')

df = pd.read_csv('./data/cleaned/foreclosures.csv')
fig = px.line(df, x='Year', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
##
col1, col2 = st.columns((2,1))

img = Image.open('subprime_mort.png')
with col1:
    st.image(img)

col2.write('''
    \n The mortgage delinquency rate is the share of the total number of mortgaged home loans in the U.S. where payment is overdue by 30 days or more. Many borrowers are eventually able to service their loan though, with foreclosure rates being generally 50-75 percent lower than delinquency rates. Total home mortgage debt in the U.S. stood at 10.94 trillion U.S. dollars in 2020.
    \n ‘Subprime’ loans, being targeted at high-risk borrowers and generally coupled with higher interest rates to compensate for the risk, have far higher delinquency rates than conventional loans. Defaulting on such loans was one of the triggers for the 2007-2010 financial crisis, with subprime delinquency rates reaching almost 26 percent around this time. These higher delinquency rates translate into higher foreclosure rates, which peaked at just under 15 percent of all subprime mortgages in 2011.
    ''')
st.write("In the second quarter of 2020, under the effects of the coronavirus crisis, the mortgage delinquency rate in the United States spiked at 8.22 percent, just one percent down from its peak of 9.3 percent during the subprime mortgage crisis of 2007-2010. Following the drastic increase directly after the outbreak of the pandemic, delinquency rates started gradually declining and reached 4.65 percent as of the fourth quarter of 2021. ")

##############################
st.header('Determining the most influencial factor')
st.write('''
To Determine the most influencial factor of all the forementioned factors,
all the data for the factors were resampled to a similar time intervals and combined into a single dataframe:
''')
df = pd.read_csv('finalcombined.csv')
st.dataframe(df)

st.write('Here CUSUSHPISA is the S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index')

desc = df.describe()
st.dataframe(desc)

corr = df.corr()
corr_inf = pd.DataFrame(corr.CSUSHPISA)

st.header("Correlation Analysis")
col1,col2 = st.columns((1,1))
col1.write('''

''')
col1.dataframe(corr_inf,  height=700)

# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corr, dtype=bool))

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
fig = plt.subplots()

sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
plt.savefig('corr_dia1.png')
img = Image.open('corr_dia1.png')
col2.image(img)

col11,col12 = st.columns((2,1))
corr_inf_new = corr_inf.head(10)
fig = px.bar(corr_inf_new)
col11.plotly_chart(fig)
col12.title("_")
col12.write('''
This analysis shows how the factors are correlated with the index and amongst themselves. Higher the bars in the graph
(in magnitude) on the left, the more correlated the factors are with the index.
\n For all the plots above the 0 value, there is a positive correlation and vice-versa.
''')

col1, col2 = st.columns((2,1))
col1.write('''
The absolute values indicate the level of correlation and the sign shows the relation, i.e either positive or negative correlation.
\n The most correlated factors(in order) are:
''')
data = abs(corr_inf_new)
sorted_data = data.sort_values(by=['CSUSHPISA'], ascending=False)
sorted_data_new = corr_inf_new.sort_values(by=['CSUSHPISA'], ascending=False)
col2.dataframe(sorted_data, height=700)
top_5 = sorted_data.head(5)

st.header("Explanatory Regression Analysis")

st.write('''
Next, a multiple regression analysis was performed on the cleaned and combined data. Given in the case, in which we would like to predict the prices
of the index, a multiple regression model would have been created and only the factor(s) influencing it the most would be chosen. Here, only the analysis
is performed in order to examine whether:\n 1) Is it better to take all the factors into consideration?
\n2) Is it better to take only the top 5 highest correlated data into consideration?
\n 3) Is it better to just take one factor into consideration?
\n Each of the cases is examined and the output is explained below.
''')
##
col1,col2 = st.columns((1,1))

col1.write('''
Case 1) Building a multiple regression model taking all the factors into consideration:
''')

col1.dataframe(corr_inf_new, height=700)

img_mlr_2 = Image.open('MLR_1.png')
col2.image(img_mlr_2)
##
col1.write('''
Case 2) The asbolute values of the correlation table have been sorted and the 5 factors with the highest correlation are represented in the table
below. When these factors(only) are taken into consideration:
''')

col1.dataframe(top_5)

img_mlr_2 = Image.open('MLR_5.png')
col2.image(img_mlr_2)
##
col1.write('''
Case 3) Only the most correlated factor i.e. the population was taken into consideration:
''')

data = abs(corr_inf_new)
sorted_data = data.sort_values(by=['CSUSHPISA'], ascending=False)
top_1 = sorted_data.head(1)
col1.dataframe(top_1)

img_mlr_2 = Image.open('MLR_O.png')
col2.image(img_mlr_2)

##
st.header("Conclusion")
st.write('''
It is known that for a regression model the R-Squared value always increases and never decres incase the number of variables are increased.
To make up for this, the Adjusted R-Squared penalises the R-Squared based on the number of variables. A very low Adjusted R-Square may indicate that some
of the variables are not contributing. And as we take more variables the R-Squared value rises, which is represented in the above cases.
However, in the case 1, the Adjusted R-Squared value is very low, indicating that there is no variable that is not contributing at all.
\n The probablistic F-Static value for case 3 is lesser than that of case 1, indicating that the most correlated field, "Population",
does have great influence on the index. This also allows us to conclude, that the results of the correlation analysis are true and are proven here.
Thus, we can say that the most import factors in order influencing the *S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index* are the most
correlated factors. And that Multiple regression analysis is only useful in quantifying relationships in factors when there is a need to do predective
modelling and not singling out one most important variable out of multiple variables.
''')

##############################
st.header('Biblography 📚')
html_string = '''<html>
    <head>
        <title> Biblography </title>
    </head>
    <body>
        <table style="border-width: 1;">
            <th style="background-color:lightgray;"> Refernces</th>
            <tr>
                <td>
                    <a href="http://fred.stlouisfed.org">fred.stlouisfed.org</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://www.statista.com">statista.com</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="www.opendoor.com">opendoor.com</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="http://nar.realtor">nar.realtor</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://tradingeconomics.com/">tradingeconomics.com</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://data.worldbank.org">data.worldbank.org</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://www.cnbc.com">cnbc.com</a>
                </td>
            </tr>
        </table>
    </body>
</html>'''
st.markdown(html_string, unsafe_allow_html=True)


# SPDX-License-Identifier: Apache-2.0

"""
What is the opset number?
=========================

.. index:: opset, target opset, version

Every library is versioned. :epkg:`scikit-learn` may change
the implementation of a specific model. That happens
for example with the `SVC <https://scikit-learn.org/stable/
modules/generated/sklearn.svm.SVC.html>`_ model where
the parameter *break_ties* was added in 0.22. :epkg:`ONNX`
does also have a version called *opset number*.
Operator *ArgMin* was added in opset 1 and changed in opset
11, 12, 13. Sometimes, it is updated to extend the list
of types it supports, sometimes, it moves a parameter
into the input list. The runtime used to deploy the model
does not implement a new version, in that case, a model
must be converted by usually using the most recent opset
supported by the runtime, we call that opset the
*targeted opset*. An ONNX graph only contains
one unique opset, every node must be described following
the specifications defined by the latest opset below the
targeted opset.

This example considers an `IsolationForest
<https://scikit-learn.org/stable/modules/generated/
sklearn.ensemble.IsolationForest.html>`_ and digs into opsets.

.. contents::
    :local:

Data
++++

A simple example.
"""
from onnx.defs import onnx_opset_version
from skl2onnx import to_onnx
import numpy
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest
from sklearn.datasets import make_blobs

X, y = make_blobs(n_samples=100, n_features=2)

model = IsolationForest(3)
model.fit(X)
labels = model.predict(X)

fig, ax = plt.subplots(1, 1)
for k in (0, 1):
    ax.plot(X[labels == k, 0], X[labels == k, 1], 'o', label="cl%d" % k)
ax.set_title("Sample")

#######################################
# ONNX
# ++++

onx = to_onnx(model, X[:1].astype(numpy.float32))
print(onx)

##########################
# The last line shows the opsets.
# Let's extract it.

domains = onx.opset_import
for dom in domains:
    print("domain: %r, version: %r" % (dom.domain, dom.version))

###################################
# There are two opsets, one for standard operators,
# the other for machine learning operators.

########################################
# ONNX and opset
# ++++++++++++++
#
# The converter can convert a model to an older opset
# than the default one, from 1 to the last available one.

def get_domain_opset(onx):
    domains = onx.opset_import
    res = [{'domain': dom.domain, 'version': dom.version}
           for dom in domains]
    return {d['domain']: d['version'] for d in res}

for opset in range(1, onnx_opset_version() + 1):
    try:
        onx = to_onnx(model, X[:1].astype(numpy.float32), target_opset=opset)
    except RuntimeError as e:
        print('target: %r error: %r' % (opset, e))
        continue
    nodes = len(onx.graph.node)
    print('target: %r --> %s %d' % (opset, get_domain_opset(onx), nodes))

########################################
# It shows that the model cannot be converted for opset
# below 5. Operator `Reshape <https://github.com/onnx/
# onnx/blob/master/docs/Operators.md#Reshape>`_ changed in
# opset 5: a parameter became an input. The converter
# does not support *opset < 5* because runtimes usually do not.
#
# Other opsets
# ++++++++++++
#
# The previous example changed the opset of the main domain
# ``''`` but the other opset domain can be changed as well.

for opset in range(9, onnx_opset_version() + 1):
    for opset_ml in range(1, 3):
        tops = {'': opset, 'ai.onnx.ml': opset_ml}
        try:
            onx = to_onnx(
                model, X[:1].astype(numpy.float32), target_opset=tops)
        except RuntimeError as e:
            print('target: %r error: %r' % (opset, e))
            continue
        nodes = len(onx.graph.node)
        print('target: %r --> %s %d' % (opset, get_domain_opset(onx), nodes))


"""
Points

TESTS::

    sage: E = EllipticCurve('37a')
    sage: P = E(0,0)
    sage: def get_points(n): return sum([point(list(i*P)[:2], size=3) for i in range(-n,n) if i != 0 and (i*P)[0] < 3])
    sage: sum([get_points(15*n).plot3d(z=n) for n in range(1,10)])
    Graphics3d Object
"""

#*****************************************************************************
#       Copyright (C) 2006 Alex Clemesha <clemesha@gmail.com>,
#                          William Stein <wstein@gmail.com>,
#                     2008 Mike Hansen <mhansen@gmail.com>,
#
#  Distributed under the terms of the GNU General Public License (GPL)
#
#    This code is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.
#
#  The full text of the GPL is available at:
#
#                  http://www.gnu.org/licenses/
#*****************************************************************************
from sage.misc.decorators import options, rename_keyword
from sage.plot.colors import to_mpl_color
from sage.plot.primitive import GraphicPrimitive_xydata
import collections

# TODO: create _allowed_options for 3D point classes to
# improve bad option handling in plot3d?
class Point(GraphicPrimitive_xydata):
    """
    Primitive class for the point graphics type.  See point?, point2d?
    or point3d? for information about actually plotting points.

    INPUT:

    - xdata - list of x values for points in Point object

    - ydata - list of y values for points in Point object

    - options - dict of valid plot options to pass to constructor

    EXAMPLES:

    Note this should normally be used indirectly via ``point`` and friends::

        sage: from sage.plot.point import Point
        sage: P = Point([1,2],[2,3],{'alpha':.5})
        sage: P
        Point set defined by 2 point(s)
        sage: P.options()['alpha']
        0.500000000000000
        sage: P.xdata
        [1, 2]

    TESTS:

    We test creating a point::

        sage: point((3,3))
        Graphics object consisting of 1 graphics primitive
    """
    def __init__(self, xdata, ydata, options):
        """
        Initializes base class Point.

        EXAMPLES::

            sage: P = point((3,4))
            sage: P[0].xdata
            [3.0]
            sage: P[0].options()['alpha']
            1
        """
        self.xdata = xdata
        self.ydata = ydata
        GraphicPrimitive_xydata.__init__(self, options)

    def _allowed_options(self):
        """
        Return the allowed options for the Point class.

        EXAMPLES::

            sage: P = point((3,4))
            sage: P[0]._allowed_options()['size']
            'How big the point is (i.e., area in points^2=(1/72 inch)^2).'
        """
        return {'alpha':'How transparent the point is.',
                'faceted': 'If True color the edge of the point. (only for 2D plots)',
                'hue':'The color given as a hue.',
                'legend_color':'The color of the legend text',
                'legend_label':'The label for this item in the legend.',
                'marker':'the marker symbol for 2D plots only (see documentation of plot() for details)',
                'markeredgecolor':'the color of the marker edge (only for 2D plots)',
                'rgbcolor':'The color as an RGB tuple.',
                'size': 'How big the point is (i.e., area in points^2=(1/72 inch)^2).',
                'zorder':'The layer level in which to draw'}

    def _plot3d_options(self, options=None):
        """
        Translate 2D plot options into 3D plot options.

        EXAMPLES::

            sage: A=point((1,1),size=22)
            sage: a=A[0];a
            Point set defined by 1 point(s)
            sage: b=a.plot3d()
            sage: b.size
            22
            sage: b=a.plot3d(size=3)
            sage: b.size
            3
        """
        if options is None:
            options = dict(self.options())
        options_3d = {}
        if 'size' in options:
            options_3d['size'] = options['size']
            del options['size']
        if options.pop('faceted', False):
            raise NotImplementedError("3D points can not be faceted.")
        for o in ('marker', 'markeredgecolor'): # remove 2D options
            if o in options:
                del options[o]

        options_3d.update(GraphicPrimitive_xydata._plot3d_options(self, options))
        return options_3d

    def plot3d(self, z=0, **kwds):
        """
        Plots a two-dimensional point in 3-D, with default height zero.

        INPUT:

        -  ``z`` - optional 3D height above `xy`-plane.  May be a list
           if self is a list of points.

        EXAMPLES:

        One point::

            sage: A=point((1,1))
            sage: a=A[0];a
            Point set defined by 1 point(s)
            sage: b=a.plot3d()

        One point with a height::

            sage: A=point((1,1))
            sage: a=A[0];a
            Point set defined by 1 point(s)
            sage: b=a.plot3d(z=3)
            sage: b.loc[2]
            3.0

        Multiple points::

            sage: P=point([(0,0), (1,1)])
            sage: p=P[0]; p
            Point set defined by 2 point(s)
            sage: q=p.plot3d(size=22)

        Multiple points with different heights::

            sage: P=point([(0,0), (1,1)])
            sage: p=P[0]
            sage: q=p.plot3d(z=[2,3])
            sage: q.all[0].loc[2]
            2.0
            sage: q.all[1].loc[2]
            3.0

        Note that keywords passed must be valid point3d options::

            sage: A=point((1,1),size=22)
            sage: a=A[0];a
            Point set defined by 1 point(s)
            sage: b=a.plot3d()
            sage: b.size
            22
            sage: b=a.plot3d(pointsize=23) # only 2D valid option
            sage: b.size
            22
            sage: b=a.plot3d(size=23) # correct keyword
            sage: b.size
            23

        TESTS:

        Heights passed as a list should have same length as
        number of points::

            sage: P=point([(0,0), (1,1), (2,3)])
            sage: p=P[0]
            sage: q=p.plot3d(z=2)
            sage: q.all[1].loc[2]
            2.0
            sage: q=p.plot3d(z=[2,-2])
            Traceback (most recent call last):
            ...
            ValueError: Incorrect number of heights given
        """
        from sage.plot.plot3d.base import Graphics3dGroup
        from sage.plot.plot3d.shapes2 import point3d
        options = self._plot3d_options()
        options.update(kwds)
        zdata=[]
        if isinstance(z, list):
            zdata=z
        else:
            zdata=[z]*len(self.xdata)
        if len(zdata)==len(self.xdata):
            all = [point3d([(x, y, z) for x, y, z in zip(self.xdata, self.ydata, zdata)], **options)]
            if len(all) == 1:
                return all[0]
            else:
                return Graphics3dGroup(all)
        else:
            raise ValueError('Incorrect number of heights given')

    def _repr_(self):
        """
        String representation of Point primitive.

        EXAMPLES::

            sage: P=point([(0,0), (1,1)])
            sage: p=P[0]; p
            Point set defined by 2 point(s)
        """
        return "Point set defined by %s point(s)"%len(self.xdata)

    def __getitem__(self, i):
        """
        Returns tuple of coordinates of point.

        EXAMPLES::

            sage: P=point([(0,0), (1,1), (2,3)])
            sage: p=P[0]; p
            Point set defined by 3 point(s)
            sage: p[1]
            (1.0, 1.0)
        """
        return self.xdata[i], self.ydata[i]

    def _render_on_subplot(self,subplot):
        r"""
        TESTS:

        We check to make sure that :trac:`2076` is fixed by verifying all
        the points are red::

            sage: point(((1,1), (2,2), (3,3)), rgbcolor=hue(1), size=30)
            Graphics object consisting of 1 graphics primitive
        """
        options = self.options()

        #Convert the color to a hex string so that the scatter
        #method does not interpret it as a list of 3 floating
        #point color specifications when there are
        #three points. This is mentioned in the matplotlib 0.98
        #documentation and fixes \#2076
        from matplotlib.colors import rgb2hex
        c = rgb2hex(to_mpl_color(options['rgbcolor']))

        a = float(options['alpha'])
        z = int(options.pop('zorder', 0))
        s = int(options['size'])
        faceted = options['faceted'] #faceted=True colors the edge of point
        markeredgecolor = options['markeredgecolor']

        scatteroptions={}
        if not faceted and markeredgecolor is None:
            scatteroptions['edgecolors'] = 'none'
        elif markeredgecolor is not None:
            scatteroptions['edgecolors'] = to_mpl_color(
                                              options.pop('markeredgecolor'))
        scatteroptions['marker'] = options.pop('marker')

        subplot.scatter(self.xdata, self.ydata, s=s, c=c, alpha=a, zorder=z,
                        label=options['legend_label'], **scatteroptions)

def point(points, **kwds):
    """
    Returns either a 2-dimensional or 3-dimensional point or sum of points.

    INPUT:

    -  ``points`` - either a single point (as a tuple), a list of
       points, a single complex number, or a list of complex numbers.

    For information regarding additional arguments, see either point2d?
    or point3d?.

    .. SEEALSO::

        :func:`sage.plot.point.point2d`, :func:`sage.plot.plot3d.shapes2.point3d`

    EXAMPLES::

        sage: point((1,2))
        Graphics object consisting of 1 graphics primitive

    ::

        sage: point((1,2,3))
        Graphics3d Object

    ::

        sage: point([(0,0), (1,1)])
        Graphics object consisting of 1 graphics primitive

    ::

        sage: point([(0,0,1), (1,1,1)])
        Graphics3d Object

    Extra options will get passed on to show(), as long as they are valid::

        sage: point([(cos(theta), sin(theta)) for theta in srange(0, 2*pi, pi/8)], frame=True)
        Graphics object consisting of 1 graphics primitive
        sage: point([(cos(theta), sin(theta)) for theta in srange(0, 2*pi, pi/8)]).show(frame=True) # These are equivalent

    TESTS:

    One can now use iterators (:trac:`13890`)::

        sage: point(iter([(1,1,1)]))
        Graphics3d Object
        sage: point(iter([(1,2),(3,5)]))
        Graphics object consisting of 1 graphics primitive
    """
    if isinstance(points, collections.Iterator):
        points = list(points)

    try:
        return point2d(points, **kwds)
    except (ValueError, TypeError):
        from sage.plot.plot3d.shapes2 import point3d
        return point3d(points, **kwds)

@rename_keyword(color='rgbcolor', pointsize='size')
@options(alpha=1, aspect_ratio='automatic', faceted=False,
        legend_color=None, legend_label=None, marker='o',
        markeredgecolor=None, rgbcolor=(0,0,1), size=10)
def point2d(points, **options):
    r"""
    A point of size ``size`` defined by point = `(x,y)`.

    INPUT:

    -  ``points`` - either a single point (as a tuple), a list of
       points, a single complex number, or a list of complex numbers.
    - ``alpha`` -- How transparent the point is.
    - ``faceted`` -- If True color the edge of the point. (only for 2D plots)
    - ``hue`` -- The color given as a hue.
    - ``legend_color`` -- The color of the legend text
    - ``legend_label`` -- The label for this item in the legend.
    - ``marker`` -- the marker symbol for 2D plots only (see documentation of
      :func:`plot` for details)
    - ``markeredgecolor`` -- the color of the marker edge (only for 2D plots)
    - ``rgbcolor`` -- The color as an RGB tuple.
    - ``size`` -- How big the point is (i.e., area in points^2=(1/72 inch)^2).
    - ``zorder`` -- The layer level in which to draw

    EXAMPLES:

    A purple point from a single tuple or coordinates::

        sage: point((0.5, 0.5), rgbcolor=hue(0.75))
        Graphics object consisting of 1 graphics primitive

    Points with customized markers and edge colors::

        sage: r = [(random(), random()) for _ in range(10)]
        sage: point(r, marker='d', markeredgecolor='red', size=20)
        Graphics object consisting of 1 graphics primitive

    Passing an empty list returns an empty plot::

        sage: point([])
        Graphics object consisting of 0 graphics primitives
        sage: import numpy; point(numpy.array([]))
        Graphics object consisting of 0 graphics primitives

    If you need a 2D point to live in 3-space later, this is possible::

        sage: A=point((1,1))
        sage: a=A[0];a
        Point set defined by 1 point(s)
        sage: b=a.plot3d(z=3)

    This is also true with multiple points::

        sage: P=point([(0,0), (1,1)])
        sage: p=P[0]
        sage: q=p.plot3d(z=[2,3])

    Here are some random larger red points, given as a list of tuples::

        sage: point(((0.5, 0.5), (1, 2), (0.5, 0.9), (-1, -1)), rgbcolor=hue(1), size=30)
        Graphics object consisting of 1 graphics primitive

    And an example with a legend::

        sage: point((0,0), rgbcolor='black', pointsize=40, legend_label='origin')
        Graphics object consisting of 1 graphics primitive

    The legend can be colored::

        sage: P = points([(0,0),(1,0)], pointsize=40, legend_label='origin', legend_color='red')
        sage: P + plot(x^2,(x,0,1), legend_label='plot', legend_color='green')
        Graphics object consisting of 2 graphics primitives

    Extra options will get passed on to show(), as long as they are valid::

        sage: point([(cos(theta), sin(theta)) for theta in srange(0, 2*pi, pi/8)], frame=True)
        Graphics object consisting of 1 graphics primitive
        sage: point([(cos(theta), sin(theta)) for theta in srange(0, 2*pi, pi/8)]).show(frame=True) # These are equivalent

    For plotting data, we can use a logarithmic scale, as long as we are sure
    not to include any nonpositive points in the logarithmic direction::

        sage: point([(1,2),(2,4),(3,4),(4,8),(4.5,32)],scale='semilogy',base=2)
        Graphics object consisting of 1 graphics primitive

    Since Sage Version 4.4 (:trac:`8599`), the size of a 2d point can be
    given by the argument ``size`` instead of ``pointsize``. The argument
    ``pointsize`` is still supported::

        sage: point((3,4), size=100)
        Graphics object consisting of 1 graphics primitive

    ::

        sage: point((3,4), pointsize=100)
        Graphics object consisting of 1 graphics primitive

    We can plot a single complex number::

        sage: point(CC(1+I), pointsize=100)
        Graphics object consisting of 1 graphics primitive

    We can also plot a list of complex numbers::

        sage: point([CC(I), CC(I+1), CC(2+2*I)], pointsize=100)
        Graphics object consisting of 1 graphics primitive

    """
    from sage.plot.plot import xydata_from_point_list
    from sage.plot.all import Graphics
    from sage.rings.all import CC, CDF
    if points in CC or points in CDF:
        pass
    else:
        try:
            l = len(points)
        except TypeError:
            # argument is an iterator
            points = list(points)
            l = len(points)

        if l == 0:
            return Graphics()

    xdata, ydata = xydata_from_point_list(points)
    g = Graphics()
    g._set_extra_kwds(Graphics._extract_kwds_for_show(options))
    g.add_primitive(Point(xdata, ydata, options))
    if options['legend_label']:
        g.legend(True)
        g._legend_colors = [options['legend_color']]
    return g

points = point